85 int scif_reserve_dma_chan(struct scif_endpt *ep)  in scif_reserve_dma_chan()  argument
93 	if (!scif_info.nodeid && scifdev_self(ep->remote_dev))  in scif_reserve_dma_chan()
98 		scifdev = ep->remote_dev;  in scif_reserve_dma_chan()
104 	mutex_lock(&ep->rma_info.rma_lock);  in scif_reserve_dma_chan()
105 	ep->rma_info.dma_chan = chan;  in scif_reserve_dma_chan()
106 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_reserve_dma_chan()
144 	struct scif_endpt *ep = mmn->ep;  in scif_rma_destroy_tcw()  local
146 	spin_lock(&ep->rma_info.tc_lock);  in scif_rma_destroy_tcw()
148 	spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_destroy_tcw()
151 static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)  in scif_rma_destroy_tcw_ep()  argument
156 	list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {  in scif_rma_destroy_tcw_ep()
162 static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)  in __scif_rma_destroy_tcw_ep()  argument
167 	spin_lock(&ep->rma_info.tc_lock);  in __scif_rma_destroy_tcw_ep()
168 	list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {  in __scif_rma_destroy_tcw_ep()
172 	spin_unlock(&ep->rma_info.tc_lock);  in __scif_rma_destroy_tcw_ep()
175 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)  in scif_rma_tc_can_cache()  argument
179 	if ((atomic_read(&ep->rma_info.tcw_total_pages)  in scif_rma_tc_can_cache()
185 			 atomic_read(&ep->rma_info.tcw_total_pages),  in scif_rma_tc_can_cache()
188 		__scif_rma_destroy_tcw_ep(ep);  in scif_rma_tc_can_cache()
234 static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)  in scif_ep_unregister_mmu_notifier()  argument
236 	struct scif_endpt_rma_info *rma = &ep->rma_info;  in scif_ep_unregister_mmu_notifier()
240 	mutex_lock(&ep->rma_info.mmn_lock);  in scif_ep_unregister_mmu_notifier()
247 	mutex_unlock(&ep->rma_info.mmn_lock);  in scif_ep_unregister_mmu_notifier()
251 				   struct mm_struct *mm, struct scif_endpt *ep)  in scif_init_mmu_notifier()  argument
253 	mmn->ep = ep;  in scif_init_mmu_notifier()
272 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)  in scif_add_mmu_notifier()  argument
280 	scif_init_mmu_notifier(mmn, current->mm, ep);  in scif_add_mmu_notifier()
285 	list_add(&mmn->list, &ep->rma_info.mmn_list);  in scif_add_mmu_notifier()
296 	struct scif_endpt *ep;  in scif_mmu_notif_handler()  local
301 		ep = list_entry(pos, struct scif_endpt, mmu_list);  in scif_mmu_notif_handler()
302 		list_del(&ep->mmu_list);  in scif_mmu_notif_handler()
304 		scif_rma_destroy_tcw_ep(ep);  in scif_mmu_notif_handler()
305 		scif_ep_unregister_mmu_notifier(ep);  in scif_mmu_notif_handler()
324 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)  in scif_add_mmu_notifier()  argument
338 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)  in scif_rma_tc_can_cache()  argument
359 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_register_temp()  local
374 	err = scif_get_window_offset(ep, 0, 0,  in scif_register_temp()
381 	*out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,  in scif_register_temp()
384 		scif_free_window_offset(ep, NULL, *out_offset);  in scif_register_temp()
394 	err = scif_map_window(ep->remote_dev, *out_window);  in scif_register_temp()
397 		scif_destroy_window(ep, *out_window);  in scif_register_temp()
405 		dev_err(&ep->remote_dev->sdev->dev,  in scif_register_temp()
585 	struct scif_endpt *ep;  in scif_rma_destroy_windows()  local
594 		ep = (struct scif_endpt *)window->ep;  in scif_rma_destroy_windows()
595 		chan = ep->rma_info.dma_chan;  in scif_rma_destroy_windows()
599 		if (!chan || !scifdev_alive(ep) ||  in scif_rma_destroy_windows()
600 		    !scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_rma_destroy_windows()
601 					 ep->rma_info.dma_chan))  in scif_rma_destroy_windows()
605 			dev_warn(&ep->remote_dev->sdev->dev,  in scif_rma_destroy_windows()
609 				scif_destroy_window(ep, window);  in scif_rma_destroy_windows()
612 			atomic_dec(&ep->rma_info.tw_refcount);  in scif_rma_destroy_windows()
629 	struct scif_endpt *ep;  in scif_rma_destroy_tcw_invalid()  local
637 		ep = (struct scif_endpt *)window->ep;  in scif_rma_destroy_tcw_invalid()
638 		chan = ep->rma_info.dma_chan;  in scif_rma_destroy_tcw_invalid()
641 		mutex_lock(&ep->rma_info.rma_lock);  in scif_rma_destroy_tcw_invalid()
642 		if (!chan || !scifdev_alive(ep) ||  in scif_rma_destroy_tcw_invalid()
643 		    !scif_drain_dma_intr(ep->remote_dev->sdev,  in scif_rma_destroy_tcw_invalid()
644 					 ep->rma_info.dma_chan)) {  in scif_rma_destroy_tcw_invalid()
646 				   &ep->rma_info.tcw_total_pages);  in scif_rma_destroy_tcw_invalid()
647 			scif_destroy_window(ep, window);  in scif_rma_destroy_tcw_invalid()
648 			atomic_dec(&ep->rma_info.tcw_refcount);  in scif_rma_destroy_tcw_invalid()
650 			dev_warn(&ep->remote_dev->sdev->dev,  in scif_rma_destroy_tcw_invalid()
653 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_destroy_tcw_invalid()
1659 	struct scif_endpt *ep = (struct scif_endpt *)epd;  in scif_rma_copy()  local
1672 	err = scif_verify_epd(ep);  in scif_rma_copy()
1680 	loopback = scifdev_self(ep->remote_dev) ? true : false;  in scif_rma_copy()
1702 	remote_req.head = &ep->rma_info.remote_reg_list;  in scif_rma_copy()
1704 	spdev = scif_get_peer_dev(ep->remote_dev);  in scif_rma_copy()
1711 		mutex_lock(&ep->rma_info.mmn_lock);  in scif_rma_copy()
1712 		mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);  in scif_rma_copy()
1714 			mmn = scif_add_mmu_notifier(current->mm, ep);  in scif_rma_copy()
1715 		mutex_unlock(&ep->rma_info.mmn_lock);  in scif_rma_copy()
1720 		cache = cache && !scif_rma_tc_can_cache(ep, len);  in scif_rma_copy()
1722 	mutex_lock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1732 			spin_lock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1734 			err = scif_query_tcw(ep, &req);  in scif_rma_copy()
1735 			spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1742 				mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1747 			atomic_inc(&ep->rma_info.tcw_refcount);  in scif_rma_copy()
1749 					  &ep->rma_info.tcw_total_pages);  in scif_rma_copy()
1751 				spin_lock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1754 				spin_unlock(&ep->rma_info.tc_lock);  in scif_rma_copy()
1770 		req.head = &ep->rma_info.reg_list;  in scif_rma_copy()
1774 			mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1782 		mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1792 	copy_work.remote_dev = ep->remote_dev;  in scif_rma_copy()
1808 		chan = ep->rma_info.dma_chan;  in scif_rma_copy()
1813 		atomic_inc(&ep->rma_info.tw_refcount);  in scif_rma_copy()
1815 	mutex_unlock(&ep->rma_info.rma_lock);  in scif_rma_copy()
1818 		struct scif_dev *rdev = ep->remote_dev;  in scif_rma_copy()
1822 						  ep->rma_info.dma_chan);  in scif_rma_copy()
1825 						  ep->rma_info.dma_chan);  in scif_rma_copy()
1835 			scif_destroy_window(ep, local_window);  in scif_rma_copy()