177 static inline u32 nwl_bridge_readl(struct nwl_pcie *pcie, u32 off)  in nwl_bridge_readl()  argument
179 	return readl(pcie->breg_base + off);  in nwl_bridge_readl()
182 static inline void nwl_bridge_writel(struct nwl_pcie *pcie, u32 val, u32 off)  in nwl_bridge_writel()  argument
184 	writel(val, pcie->breg_base + off);  in nwl_bridge_writel()
187 static bool nwl_pcie_link_up(struct nwl_pcie *pcie)  in nwl_pcie_link_up()  argument
189 	if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PCIE_PHY_LINKUP_BIT)  in nwl_pcie_link_up()
194 static bool nwl_phy_link_up(struct nwl_pcie *pcie)  in nwl_phy_link_up()  argument
196 	if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PHY_RDY_LINKUP_BIT)  in nwl_phy_link_up()
201 static int nwl_wait_for_link(struct nwl_pcie *pcie)  in nwl_wait_for_link()  argument
203 	struct device *dev = pcie->dev;  in nwl_wait_for_link()
208 		if (nwl_phy_link_up(pcie))  in nwl_wait_for_link()
219 	struct nwl_pcie *pcie = bus->sysdata;  in nwl_pcie_valid_device()  local
222 	if (bus->number != pcie->root_busno) {  in nwl_pcie_valid_device()
223 		if (!nwl_pcie_link_up(pcie))  in nwl_pcie_valid_device()
228 	if (bus->number == pcie->root_busno && devfn > 0)  in nwl_pcie_valid_device()
247 	struct nwl_pcie *pcie = bus->sysdata;  in nwl_pcie_map_bus()  local
256 	return pcie->ecam_base + relbus + where;  in nwl_pcie_map_bus()
268 	struct nwl_pcie *pcie = data;  in nwl_pcie_misc_handler()  local
269 	struct device *dev = pcie->dev;  in nwl_pcie_misc_handler()
273 	misc_stat = nwl_bridge_readl(pcie, MSGF_MISC_STATUS) &  in nwl_pcie_misc_handler()
318 	nwl_bridge_writel(pcie, misc_stat, MSGF_MISC_STATUS);  in nwl_pcie_misc_handler()
326 	struct nwl_pcie *pcie;  in nwl_pcie_leg_handler()  local
332 	pcie = irq_desc_get_handler_data(desc);  in nwl_pcie_leg_handler()
334 	while ((status = nwl_bridge_readl(pcie, MSGF_LEG_STATUS) &  in nwl_pcie_leg_handler()
337 			virq = irq_find_mapping(pcie->legacy_irq_domain, bit);  in nwl_pcie_leg_handler()
346 static void nwl_pcie_handle_msi_irq(struct nwl_pcie *pcie, u32 status_reg)  in nwl_pcie_handle_msi_irq()  argument
353 	msi = &pcie->msi;  in nwl_pcie_handle_msi_irq()
355 	while ((status = nwl_bridge_readl(pcie, status_reg)) != 0) {  in nwl_pcie_handle_msi_irq()
357 			nwl_bridge_writel(pcie, 1 << bit, status_reg);  in nwl_pcie_handle_msi_irq()
368 	struct nwl_pcie *pcie = irq_desc_get_handler_data(desc);  in nwl_pcie_msi_handler_high()  local
371 	nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_HI);  in nwl_pcie_msi_handler_high()
378 	struct nwl_pcie *pcie = irq_desc_get_handler_data(desc);  in nwl_pcie_msi_handler_low()  local
381 	nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_LO);  in nwl_pcie_msi_handler_low()
388 	struct nwl_pcie *pcie;  in nwl_mask_leg_irq()  local
393 	pcie = irq_desc_get_chip_data(desc);  in nwl_mask_leg_irq()
395 	raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags);  in nwl_mask_leg_irq()
396 	val = nwl_bridge_readl(pcie, MSGF_LEG_MASK);  in nwl_mask_leg_irq()
397 	nwl_bridge_writel(pcie, (val & (~mask)), MSGF_LEG_MASK);  in nwl_mask_leg_irq()
398 	raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags);  in nwl_mask_leg_irq()
404 	struct nwl_pcie *pcie;  in nwl_unmask_leg_irq()  local
409 	pcie = irq_desc_get_chip_data(desc);  in nwl_unmask_leg_irq()
411 	raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags);  in nwl_unmask_leg_irq()
412 	val = nwl_bridge_readl(pcie, MSGF_LEG_MASK);  in nwl_unmask_leg_irq()
413 	nwl_bridge_writel(pcie, (val | mask), MSGF_LEG_MASK);  in nwl_unmask_leg_irq()
414 	raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags);  in nwl_unmask_leg_irq()
459 	struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data);  in nwl_compose_msi_msg()  local
460 	phys_addr_t msi_addr = pcie->phys_pcie_reg_base;  in nwl_compose_msi_msg()
482 	struct nwl_pcie *pcie = domain->host_data;  in nwl_irq_domain_alloc()  local
483 	struct nwl_msi *msi = &pcie->msi;  in nwl_irq_domain_alloc()
508 	struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data);  in nwl_irq_domain_free()  local
509 	struct nwl_msi *msi = &pcie->msi;  in nwl_irq_domain_free()
522 static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie)  in nwl_pcie_init_msi_irq_domain()  argument
525 	struct device *dev = pcie->dev;  in nwl_pcie_init_msi_irq_domain()
527 	struct nwl_msi *msi = &pcie->msi;  in nwl_pcie_init_msi_irq_domain()
530 						&dev_msi_domain_ops, pcie);  in nwl_pcie_init_msi_irq_domain()
547 static int nwl_pcie_init_irq_domain(struct nwl_pcie *pcie)  in nwl_pcie_init_irq_domain()  argument
549 	struct device *dev = pcie->dev;  in nwl_pcie_init_irq_domain()
559 	pcie->legacy_irq_domain = irq_domain_add_linear(legacy_intc_node,  in nwl_pcie_init_irq_domain()
562 							pcie);  in nwl_pcie_init_irq_domain()
564 	if (!pcie->legacy_irq_domain) {  in nwl_pcie_init_irq_domain()
569 	raw_spin_lock_init(&pcie->leg_mask_lock);  in nwl_pcie_init_irq_domain()
570 	nwl_pcie_init_msi_irq_domain(pcie);  in nwl_pcie_init_irq_domain()
574 static int nwl_pcie_enable_msi(struct nwl_pcie *pcie)  in nwl_pcie_enable_msi()  argument
576 	struct device *dev = pcie->dev;  in nwl_pcie_enable_msi()
578 	struct nwl_msi *msi = &pcie->msi;  in nwl_pcie_enable_msi()
598 					 nwl_pcie_msi_handler_high, pcie);  in nwl_pcie_enable_msi()
609 					 nwl_pcie_msi_handler_low, pcie);  in nwl_pcie_enable_msi()
612 	ret = nwl_bridge_readl(pcie, I_MSII_CAPABILITIES) & MSII_PRESENT;  in nwl_pcie_enable_msi()
620 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) |  in nwl_pcie_enable_msi()
624 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) |  in nwl_pcie_enable_msi()
628 	base = pcie->phys_pcie_reg_base;  in nwl_pcie_enable_msi()
629 	nwl_bridge_writel(pcie, lower_32_bits(base), I_MSII_BASE_LO);  in nwl_pcie_enable_msi()
630 	nwl_bridge_writel(pcie, upper_32_bits(base), I_MSII_BASE_HI);  in nwl_pcie_enable_msi()
636 	nwl_bridge_writel(pcie, 0, MSGF_MSI_MASK_HI);  in nwl_pcie_enable_msi()
638 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie,  MSGF_MSI_STATUS_HI) &  in nwl_pcie_enable_msi()
641 	nwl_bridge_writel(pcie, MSGF_MSI_SR_HI_MASK, MSGF_MSI_MASK_HI);  in nwl_pcie_enable_msi()
647 	nwl_bridge_writel(pcie, 0, MSGF_MSI_MASK_LO);  in nwl_pcie_enable_msi()
649 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MSI_STATUS_LO) &  in nwl_pcie_enable_msi()
652 	nwl_bridge_writel(pcie, MSGF_MSI_SR_LO_MASK, MSGF_MSI_MASK_LO);  in nwl_pcie_enable_msi()
661 static int nwl_pcie_bridge_init(struct nwl_pcie *pcie)  in nwl_pcie_bridge_init()  argument
663 	struct device *dev = pcie->dev;  in nwl_pcie_bridge_init()
668 	breg_val = nwl_bridge_readl(pcie, E_BREG_CAPABILITIES) & BREG_PRESENT;  in nwl_pcie_bridge_init()
675 	nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_breg_base),  in nwl_pcie_bridge_init()
677 	nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_breg_base),  in nwl_pcie_bridge_init()
681 	nwl_bridge_writel(pcie, ~BREG_ENABLE_FORCE & BREG_ENABLE,  in nwl_pcie_bridge_init()
685 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_PCIE_RX0) |  in nwl_pcie_bridge_init()
689 	nwl_bridge_writel(pcie, SET_ISUB_CONTROL, I_ISUB_CONTROL);  in nwl_pcie_bridge_init()
692 	nwl_bridge_writel(pcie, CFG_ENABLE_MSG_FILTER_MASK,  in nwl_pcie_bridge_init()
695 	err = nwl_wait_for_link(pcie);  in nwl_pcie_bridge_init()
699 	ecam_val = nwl_bridge_readl(pcie, E_ECAM_CAPABILITIES) & E_ECAM_PRESENT;  in nwl_pcie_bridge_init()
706 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, E_ECAM_CONTROL) |  in nwl_pcie_bridge_init()
709 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, E_ECAM_CONTROL) |  in nwl_pcie_bridge_init()
710 			  (pcie->ecam_value << E_ECAM_SIZE_SHIFT),  in nwl_pcie_bridge_init()
713 	nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_ecam_base),  in nwl_pcie_bridge_init()
715 	nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_ecam_base),  in nwl_pcie_bridge_init()
719 	ecam_val = nwl_bridge_readl(pcie, E_ECAM_CONTROL);  in nwl_pcie_bridge_init()
720 	pcie->last_busno = (ecam_val & E_ECAM_SIZE_LOC) >> E_ECAM_SIZE_SHIFT;  in nwl_pcie_bridge_init()
724 	ecam_val |= (pcie->last_busno << E_ECAM_SIZE_SHIFT);  in nwl_pcie_bridge_init()
725 	writel(ecam_val, (pcie->ecam_base + PCI_PRIMARY_BUS));  in nwl_pcie_bridge_init()
727 	if (nwl_pcie_link_up(pcie))  in nwl_pcie_bridge_init()
733 	pcie->irq_misc = platform_get_irq_byname(pdev, "misc");  in nwl_pcie_bridge_init()
734 	if (pcie->irq_misc < 0) {  in nwl_pcie_bridge_init()
736 			pcie->irq_misc);  in nwl_pcie_bridge_init()
740 	err = devm_request_irq(dev, pcie->irq_misc,  in nwl_pcie_bridge_init()
742 			       "nwl_pcie:misc", pcie);  in nwl_pcie_bridge_init()
745 			pcie->irq_misc);  in nwl_pcie_bridge_init()
750 	nwl_bridge_writel(pcie, (u32)~MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK);  in nwl_pcie_bridge_init()
753 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MISC_STATUS) &  in nwl_pcie_bridge_init()
757 	nwl_bridge_writel(pcie, MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK);  in nwl_pcie_bridge_init()
761 	nwl_bridge_writel(pcie, (u32)~MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK);  in nwl_pcie_bridge_init()
764 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_LEG_STATUS) &  in nwl_pcie_bridge_init()
768 	nwl_bridge_writel(pcie, MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK);  in nwl_pcie_bridge_init()
771 	nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_INTERRUPT) |  in nwl_pcie_bridge_init()
777 static int nwl_pcie_parse_dt(struct nwl_pcie *pcie,  in nwl_pcie_parse_dt()  argument
780 	struct device *dev = pcie->dev;  in nwl_pcie_parse_dt()
793 	pcie->breg_base = devm_ioremap_resource(dev, res);  in nwl_pcie_parse_dt()
794 	if (IS_ERR(pcie->breg_base))  in nwl_pcie_parse_dt()
795 		return PTR_ERR(pcie->breg_base);  in nwl_pcie_parse_dt()
796 	pcie->phys_breg_base = res->start;  in nwl_pcie_parse_dt()
799 	pcie->pcireg_base = devm_ioremap_resource(dev, res);  in nwl_pcie_parse_dt()
800 	if (IS_ERR(pcie->pcireg_base))  in nwl_pcie_parse_dt()
801 		return PTR_ERR(pcie->pcireg_base);  in nwl_pcie_parse_dt()
802 	pcie->phys_pcie_reg_base = res->start;  in nwl_pcie_parse_dt()
805 	pcie->ecam_base = devm_pci_remap_cfg_resource(dev, res);  in nwl_pcie_parse_dt()
806 	if (IS_ERR(pcie->ecam_base))  in nwl_pcie_parse_dt()
807 		return PTR_ERR(pcie->ecam_base);  in nwl_pcie_parse_dt()
808 	pcie->phys_ecam_base = res->start;  in nwl_pcie_parse_dt()
811 	pcie->irq_intx = platform_get_irq_byname(pdev, "intx");  in nwl_pcie_parse_dt()
812 	if (pcie->irq_intx < 0) {  in nwl_pcie_parse_dt()
813 		dev_err(dev, "failed to get intx IRQ %d\n", pcie->irq_intx);  in nwl_pcie_parse_dt()
814 		return pcie->irq_intx;  in nwl_pcie_parse_dt()
817 	irq_set_chained_handler_and_data(pcie->irq_intx,  in nwl_pcie_parse_dt()
818 					 nwl_pcie_leg_handler, pcie);  in nwl_pcie_parse_dt()
831 	struct nwl_pcie *pcie;  in nwl_pcie_probe()  local
839 	bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));  in nwl_pcie_probe()
843 	pcie = pci_host_bridge_priv(bridge);  in nwl_pcie_probe()
845 	pcie->dev = dev;  in nwl_pcie_probe()
846 	pcie->ecam_value = NWL_ECAM_VALUE_DEFAULT;  in nwl_pcie_probe()
848 	err = nwl_pcie_parse_dt(pcie, pdev);  in nwl_pcie_probe()
854 	pcie->clk = devm_clk_get(dev, NULL);  in nwl_pcie_probe()
855 	if (IS_ERR(pcie->clk))  in nwl_pcie_probe()
856 		return PTR_ERR(pcie->clk);  in nwl_pcie_probe()
858 	err = clk_prepare_enable(pcie->clk);  in nwl_pcie_probe()
864 	err = nwl_pcie_bridge_init(pcie);  in nwl_pcie_probe()
881 	err = nwl_pcie_init_irq_domain(pcie);  in nwl_pcie_probe()
889 	bridge->sysdata = pcie;  in nwl_pcie_probe()
890 	bridge->busnr = pcie->root_busno;  in nwl_pcie_probe()
896 		err = nwl_pcie_enable_msi(pcie);  in nwl_pcie_probe()