61 static int gen3_poll_link(struct intel_ntb_dev *ndev);
93 static int gen3_poll_link(struct intel_ntb_dev *ndev)  in gen3_poll_link()  argument
98 	ndev->reg->db_iowrite(ndev->db_link_mask,  in gen3_poll_link()
99 			      ndev->self_mmio +  in gen3_poll_link()
100 			      ndev->self_reg->db_clear);  in gen3_poll_link()
102 	rc = pci_read_config_word(ndev->ntb.pdev,  in gen3_poll_link()
107 	if (reg_val == ndev->lnk_sta)  in gen3_poll_link()
110 	ndev->lnk_sta = reg_val;  in gen3_poll_link()
115 static int gen3_init_isr(struct intel_ntb_dev *ndev)  in gen3_init_isr()  argument
127 		iowrite8(i, ndev->self_mmio + GEN3_INTVEC_OFFSET + i);  in gen3_init_isr()
130 	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {  in gen3_init_isr()
132 			 ndev->self_mmio + GEN3_INTVEC_OFFSET +  in gen3_init_isr()
136 	return ndev_init_isr(ndev, GEN3_DB_MSIX_VECTOR_COUNT,  in gen3_init_isr()
142 static int gen3_setup_b2b_mw(struct intel_ntb_dev *ndev,  in gen3_setup_b2b_mw()  argument
150 	pdev = ndev->ntb.pdev;  in gen3_setup_b2b_mw()
151 	mmio = ndev->self_mmio;  in gen3_setup_b2b_mw()
168 	ndev->peer_mmio = ndev->self_mmio;  in gen3_setup_b2b_mw()
173 static int gen3_init_ntb(struct intel_ntb_dev *ndev)  in gen3_init_ntb()  argument
178 	ndev->mw_count = XEON_MW_COUNT;  in gen3_init_ntb()
179 	ndev->spad_count = GEN3_SPAD_COUNT;  in gen3_init_ntb()
180 	ndev->db_count = GEN3_DB_COUNT;  in gen3_init_ntb()
181 	ndev->db_link_mask = GEN3_DB_LINK_BIT;  in gen3_init_ntb()
184 	if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)  in gen3_init_ntb()
185 		ndev->db_link_mask |= BIT_ULL(31);  in gen3_init_ntb()
187 	switch (ndev->ntb.topo) {  in gen3_init_ntb()
190 		ndev->self_reg = &gen3_pri_reg;  in gen3_init_ntb()
191 		ndev->peer_reg = &gen3_b2b_reg;  in gen3_init_ntb()
192 		ndev->xlat_reg = &gen3_sec_xlat;  in gen3_init_ntb()
194 		if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {  in gen3_init_ntb()
195 			rc = gen3_setup_b2b_mw(ndev,  in gen3_init_ntb()
199 			rc = gen3_setup_b2b_mw(ndev,  in gen3_init_ntb()
209 			  ndev->self_mmio + GEN3_SPCICMD_OFFSET);  in gen3_init_ntb()
217 	ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;  in gen3_init_ntb()
219 	ndev->reg->db_iowrite(ndev->db_valid_mask,  in gen3_init_ntb()
220 			      ndev->self_mmio +  in gen3_init_ntb()
221 			      ndev->self_reg->db_mask);  in gen3_init_ntb()
226 int gen3_init_dev(struct intel_ntb_dev *ndev)  in gen3_init_dev()  argument
232 	pdev = ndev->ntb.pdev;  in gen3_init_dev()
234 	ndev->reg = &gen3_reg;  in gen3_init_dev()
240 	ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);  in gen3_init_dev()
242 		ntb_topo_string(ndev->ntb.topo));  in gen3_init_dev()
243 	if (ndev->ntb.topo == NTB_TOPO_NONE)  in gen3_init_dev()
246 	ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;  in gen3_init_dev()
248 	rc = gen3_init_ntb(ndev);  in gen3_init_dev()
252 	return gen3_init_isr(ndev);  in gen3_init_dev()
258 	struct intel_ntb_dev *ndev;  in ndev_ntb3_debugfs_read()  local
265 	ndev = filp->private_data;  in ndev_ntb3_debugfs_read()
266 	mmio = ndev->self_mmio;  in ndev_ntb3_debugfs_read()
281 			 ntb_topo_string(ndev->ntb.topo));  in ndev_ntb3_debugfs_read()
284 			 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);  in ndev_ntb3_debugfs_read()
286 			 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);  in ndev_ntb3_debugfs_read()
288 	if (!ndev->reg->link_is_up(ndev))  in ndev_ntb3_debugfs_read()
296 				 NTB_LNK_STA_SPEED(ndev->lnk_sta));  in ndev_ntb3_debugfs_read()
299 				 NTB_LNK_STA_WIDTH(ndev->lnk_sta));  in ndev_ntb3_debugfs_read()
303 			 "Memory Window Count -\t%u\n", ndev->mw_count);  in ndev_ntb3_debugfs_read()
305 			 "Scratchpad Count -\t%u\n", ndev->spad_count);  in ndev_ntb3_debugfs_read()
307 			 "Doorbell Count -\t%u\n", ndev->db_count);  in ndev_ntb3_debugfs_read()
309 			 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);  in ndev_ntb3_debugfs_read()
311 			 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);  in ndev_ntb3_debugfs_read()
314 			 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);  in ndev_ntb3_debugfs_read()
316 			 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);  in ndev_ntb3_debugfs_read()
318 			 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);  in ndev_ntb3_debugfs_read()
320 	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);  in ndev_ntb3_debugfs_read()
324 	u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);  in ndev_ntb3_debugfs_read()
347 	if (ntb_topo_is_b2b(ndev->ntb.topo)) {  in ndev_ntb3_debugfs_read()
393 	if (!pci_read_config_word(ndev->ntb.pdev,  in ndev_ntb3_debugfs_read()
398 	if (!pci_read_config_word(ndev->ntb.pdev,  in ndev_ntb3_debugfs_read()
403 	if (!pci_read_config_dword(ndev->ntb.pdev,  in ndev_ntb3_debugfs_read()
408 	if (!pci_read_config_dword(ndev->ntb.pdev,  in ndev_ntb3_debugfs_read()
422 	struct intel_ntb_dev *ndev;  in intel_ntb3_link_enable()  local
425 	ndev = container_of(ntb, struct intel_ntb_dev, ntb);  in intel_ntb3_link_enable()
436 	ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);  in intel_ntb3_link_enable()
440 	iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);  in intel_ntb3_link_enable()
447 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);  in intel_ntb3_mw_set_trans()  local
457 	if (idx >= ndev->b2b_idx && !ndev->b2b_off)  in intel_ntb3_mw_set_trans()
460 	bar = ndev_mw_to_bar(ndev, idx);  in intel_ntb3_mw_set_trans()
464 	bar_size = pci_resource_len(ndev->ntb.pdev, bar);  in intel_ntb3_mw_set_trans()
466 	if (idx == ndev->b2b_idx)  in intel_ntb3_mw_set_trans()
467 		mw_size = bar_size - ndev->b2b_off;  in intel_ntb3_mw_set_trans()
479 	mmio = ndev->self_mmio;  in intel_ntb3_mw_set_trans()
480 	xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);  in intel_ntb3_mw_set_trans()
481 	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);  in intel_ntb3_mw_set_trans()
482 	base = pci_resource_start(ndev->ntb.pdev, bar);  in intel_ntb3_mw_set_trans()
512 	limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;  in intel_ntb3_mw_set_trans()
537 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);  in intel_ntb3_peer_db_set()  local
540 	if (db_bits & ~ndev->db_valid_mask)  in intel_ntb3_peer_db_set()
545 		iowrite32(1, ndev->peer_mmio +  in intel_ntb3_peer_db_set()
546 				ndev->peer_reg->db_bell + (bit * 4));  in intel_ntb3_peer_db_set()
555 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);  in intel_ntb3_db_read()  local
557 	return ndev_db_read(ndev,  in intel_ntb3_db_read()
558 			    ndev->self_mmio +  in intel_ntb3_db_read()
559 			    ndev->self_reg->db_clear);  in intel_ntb3_db_read()
564 	struct intel_ntb_dev *ndev = ntb_ndev(ntb);  in intel_ntb3_db_clear()  local
566 	return ndev_db_write(ndev, db_bits,  in intel_ntb3_db_clear()
567 			     ndev->self_mmio +  in intel_ntb3_db_clear()
568 			     ndev->self_reg->db_clear);  in intel_ntb3_db_clear()