1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright 2017 IBM Corp.
3 #include <linux/sched/mm.h>
4 #include <linux/mutex.h>
5 #include <linux/mm_types.h>
6 #include <linux/mmu_context.h>
7 #include <asm/copro.h>
8 #include <asm/pnv-ocxl.h>
9 #include <misc/ocxl.h>
10 #include "ocxl_internal.h"
11 #include "trace.h"
12 
13 
14 #define SPA_PASID_BITS		15
15 #define SPA_PASID_MAX		((1 << SPA_PASID_BITS) - 1)
16 #define SPA_PE_MASK		SPA_PASID_MAX
17 #define SPA_SPA_SIZE_LOG	22 /* Each SPA is 4 Mb */
18 
19 #define SPA_CFG_SF		(1ull << (63-0))
20 #define SPA_CFG_TA		(1ull << (63-1))
21 #define SPA_CFG_HV		(1ull << (63-3))
22 #define SPA_CFG_UV		(1ull << (63-4))
23 #define SPA_CFG_XLAT_hpt	(0ull << (63-6)) /* Hashed page table (HPT) mode */
24 #define SPA_CFG_XLAT_roh	(2ull << (63-6)) /* Radix on HPT mode */
25 #define SPA_CFG_XLAT_ror	(3ull << (63-6)) /* Radix on Radix mode */
26 #define SPA_CFG_PR		(1ull << (63-49))
27 #define SPA_CFG_TC		(1ull << (63-54))
28 #define SPA_CFG_DR		(1ull << (63-59))
29 
30 #define SPA_XSL_TF		(1ull << (63-3))  /* Translation fault */
31 #define SPA_XSL_S		(1ull << (63-38)) /* Store operation */
32 
33 #define SPA_PE_VALID		0x80000000
34 
35 
36 struct pe_data {
37 	struct mm_struct *mm;
38 	/* callback to trigger when a translation fault occurs */
39 	void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
40 	/* opaque pointer to be passed to the above callback */
41 	void *xsl_err_data;
42 	struct rcu_head rcu;
43 };
44 
45 struct spa {
46 	struct ocxl_process_element *spa_mem;
47 	int spa_order;
48 	struct mutex spa_lock;
49 	struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
50 	char *irq_name;
51 	int virq;
52 	void __iomem *reg_dsisr;
53 	void __iomem *reg_dar;
54 	void __iomem *reg_tfc;
55 	void __iomem *reg_pe_handle;
56 	/*
57 	 * The following field are used by the memory fault
58 	 * interrupt handler. We can only have one interrupt at a
59 	 * time. The NPU won't raise another interrupt until the
60 	 * previous one has been ack'd by writing to the TFC register
61 	 */
62 	struct xsl_fault {
63 		struct work_struct fault_work;
64 		u64 pe;
65 		u64 dsisr;
66 		u64 dar;
67 		struct pe_data pe_data;
68 	} xsl_fault;
69 };
70 
71 /*
72  * A opencapi link can be used be by several PCI functions. We have
73  * one link per device slot.
74  *
75  * A linked list of opencapi links should suffice, as there's a
76  * limited number of opencapi slots on a system and lookup is only
77  * done when the device is probed
78  */
79 struct link {
80 	struct list_head list;
81 	struct kref ref;
82 	int domain;
83 	int bus;
84 	int dev;
85 	atomic_t irq_available;
86 	struct spa *spa;
87 	void *platform_data;
88 };
89 static struct list_head links_list = LIST_HEAD_INIT(links_list);
90 static DEFINE_MUTEX(links_list_lock);
91 
92 enum xsl_response {
93 	CONTINUE,
94 	ADDRESS_ERROR,
95 	RESTART,
96 };
97 
98 
read_irq(struct spa * spa,u64 * dsisr,u64 * dar,u64 * pe)99 static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
100 {
101 	u64 reg;
102 
103 	*dsisr = in_be64(spa->reg_dsisr);
104 	*dar = in_be64(spa->reg_dar);
105 	reg = in_be64(spa->reg_pe_handle);
106 	*pe = reg & SPA_PE_MASK;
107 }
108 
ack_irq(struct spa * spa,enum xsl_response r)109 static void ack_irq(struct spa *spa, enum xsl_response r)
110 {
111 	u64 reg = 0;
112 
113 	/* continue is not supported */
114 	if (r == RESTART)
115 		reg = PPC_BIT(31);
116 	else if (r == ADDRESS_ERROR)
117 		reg = PPC_BIT(30);
118 	else
119 		WARN(1, "Invalid irq response %d\n", r);
120 
121 	if (reg) {
122 		trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
123 				spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
124 		out_be64(spa->reg_tfc, reg);
125 	}
126 }
127 
xsl_fault_handler_bh(struct work_struct * fault_work)128 static void xsl_fault_handler_bh(struct work_struct *fault_work)
129 {
130 	vm_fault_t flt = 0;
131 	unsigned long access, flags, inv_flags = 0;
132 	enum xsl_response r;
133 	struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
134 					fault_work);
135 	struct spa *spa = container_of(fault, struct spa, xsl_fault);
136 
137 	int rc;
138 
139 	/*
140 	 * We must release a reference on mm_users whenever exiting this
141 	 * function (taken in the memory fault interrupt handler)
142 	 */
143 	rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
144 				&flt);
145 	if (rc) {
146 		pr_debug("copro_handle_mm_fault failed: %d\n", rc);
147 		if (fault->pe_data.xsl_err_cb) {
148 			fault->pe_data.xsl_err_cb(
149 				fault->pe_data.xsl_err_data,
150 				fault->dar, fault->dsisr);
151 		}
152 		r = ADDRESS_ERROR;
153 		goto ack;
154 	}
155 
156 	if (!radix_enabled()) {
157 		/*
158 		 * update_mmu_cache() will not have loaded the hash
159 		 * since current->trap is not a 0x400 or 0x300, so
160 		 * just call hash_page_mm() here.
161 		 */
162 		access = _PAGE_PRESENT | _PAGE_READ;
163 		if (fault->dsisr & SPA_XSL_S)
164 			access |= _PAGE_WRITE;
165 
166 		if (REGION_ID(fault->dar) != USER_REGION_ID)
167 			access |= _PAGE_PRIVILEGED;
168 
169 		local_irq_save(flags);
170 		hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
171 			inv_flags);
172 		local_irq_restore(flags);
173 	}
174 	r = RESTART;
175 ack:
176 	mmput(fault->pe_data.mm);
177 	ack_irq(spa, r);
178 }
179 
xsl_fault_handler(int irq,void * data)180 static irqreturn_t xsl_fault_handler(int irq, void *data)
181 {
182 	struct link *link = (struct link *) data;
183 	struct spa *spa = link->spa;
184 	u64 dsisr, dar, pe_handle;
185 	struct pe_data *pe_data;
186 	struct ocxl_process_element *pe;
187 	int lpid, pid, tid;
188 	bool schedule = false;
189 
190 	read_irq(spa, &dsisr, &dar, &pe_handle);
191 	trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
192 
193 	WARN_ON(pe_handle > SPA_PE_MASK);
194 	pe = spa->spa_mem + pe_handle;
195 	lpid = be32_to_cpu(pe->lpid);
196 	pid = be32_to_cpu(pe->pid);
197 	tid = be32_to_cpu(pe->tid);
198 	/* We could be reading all null values here if the PE is being
199 	 * removed while an interrupt kicks in. It's not supposed to
200 	 * happen if the driver notified the AFU to terminate the
201 	 * PASID, and the AFU waited for pending operations before
202 	 * acknowledging. But even if it happens, we won't find a
203 	 * memory context below and fail silently, so it should be ok.
204 	 */
205 	if (!(dsisr & SPA_XSL_TF)) {
206 		WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
207 		ack_irq(spa, ADDRESS_ERROR);
208 		return IRQ_HANDLED;
209 	}
210 
211 	rcu_read_lock();
212 	pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
213 	if (!pe_data) {
214 		/*
215 		 * Could only happen if the driver didn't notify the
216 		 * AFU about PASID termination before removing the PE,
217 		 * or the AFU didn't wait for all memory access to
218 		 * have completed.
219 		 *
220 		 * Either way, we fail early, but we shouldn't log an
221 		 * error message, as it is a valid (if unexpected)
222 		 * scenario
223 		 */
224 		rcu_read_unlock();
225 		pr_debug("Unknown mm context for xsl interrupt\n");
226 		ack_irq(spa, ADDRESS_ERROR);
227 		return IRQ_HANDLED;
228 	}
229 	WARN_ON(pe_data->mm->context.id != pid);
230 
231 	if (mmget_not_zero(pe_data->mm)) {
232 			spa->xsl_fault.pe = pe_handle;
233 			spa->xsl_fault.dar = dar;
234 			spa->xsl_fault.dsisr = dsisr;
235 			spa->xsl_fault.pe_data = *pe_data;
236 			schedule = true;
237 			/* mm_users count released by bottom half */
238 	}
239 	rcu_read_unlock();
240 	if (schedule)
241 		schedule_work(&spa->xsl_fault.fault_work);
242 	else
243 		ack_irq(spa, ADDRESS_ERROR);
244 	return IRQ_HANDLED;
245 }
246 
unmap_irq_registers(struct spa * spa)247 static void unmap_irq_registers(struct spa *spa)
248 {
249 	pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
250 				spa->reg_pe_handle);
251 }
252 
map_irq_registers(struct pci_dev * dev,struct spa * spa)253 static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
254 {
255 	return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
256 				&spa->reg_tfc, &spa->reg_pe_handle);
257 }
258 
setup_xsl_irq(struct pci_dev * dev,struct link * link)259 static int setup_xsl_irq(struct pci_dev *dev, struct link *link)
260 {
261 	struct spa *spa = link->spa;
262 	int rc;
263 	int hwirq;
264 
265 	rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
266 	if (rc)
267 		return rc;
268 
269 	rc = map_irq_registers(dev, spa);
270 	if (rc)
271 		return rc;
272 
273 	spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
274 				link->domain, link->bus, link->dev);
275 	if (!spa->irq_name) {
276 		unmap_irq_registers(spa);
277 		dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
278 		return -ENOMEM;
279 	}
280 	/*
281 	 * At some point, we'll need to look into allowing a higher
282 	 * number of interrupts. Could we have an IRQ domain per link?
283 	 */
284 	spa->virq = irq_create_mapping(NULL, hwirq);
285 	if (!spa->virq) {
286 		kfree(spa->irq_name);
287 		unmap_irq_registers(spa);
288 		dev_err(&dev->dev,
289 			"irq_create_mapping failed for translation interrupt\n");
290 		return -EINVAL;
291 	}
292 
293 	dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
294 
295 	rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
296 			link);
297 	if (rc) {
298 		irq_dispose_mapping(spa->virq);
299 		kfree(spa->irq_name);
300 		unmap_irq_registers(spa);
301 		dev_err(&dev->dev,
302 			"request_irq failed for translation interrupt: %d\n",
303 			rc);
304 		return -EINVAL;
305 	}
306 	return 0;
307 }
308 
release_xsl_irq(struct link * link)309 static void release_xsl_irq(struct link *link)
310 {
311 	struct spa *spa = link->spa;
312 
313 	if (spa->virq) {
314 		free_irq(spa->virq, link);
315 		irq_dispose_mapping(spa->virq);
316 	}
317 	kfree(spa->irq_name);
318 	unmap_irq_registers(spa);
319 }
320 
alloc_spa(struct pci_dev * dev,struct link * link)321 static int alloc_spa(struct pci_dev *dev, struct link *link)
322 {
323 	struct spa *spa;
324 
325 	spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
326 	if (!spa)
327 		return -ENOMEM;
328 
329 	mutex_init(&spa->spa_lock);
330 	INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
331 	INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
332 
333 	spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
334 	spa->spa_mem = (struct ocxl_process_element *)
335 		__get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
336 	if (!spa->spa_mem) {
337 		dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
338 		kfree(spa);
339 		return -ENOMEM;
340 	}
341 	pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
342 		link->dev, spa->spa_mem);
343 
344 	link->spa = spa;
345 	return 0;
346 }
347 
free_spa(struct link * link)348 static void free_spa(struct link *link)
349 {
350 	struct spa *spa = link->spa;
351 
352 	pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
353 		link->dev);
354 
355 	if (spa && spa->spa_mem) {
356 		free_pages((unsigned long) spa->spa_mem, spa->spa_order);
357 		kfree(spa);
358 		link->spa = NULL;
359 	}
360 }
361 
alloc_link(struct pci_dev * dev,int PE_mask,struct link ** out_link)362 static int alloc_link(struct pci_dev *dev, int PE_mask, struct link **out_link)
363 {
364 	struct link *link;
365 	int rc;
366 
367 	link = kzalloc(sizeof(struct link), GFP_KERNEL);
368 	if (!link)
369 		return -ENOMEM;
370 
371 	kref_init(&link->ref);
372 	link->domain = pci_domain_nr(dev->bus);
373 	link->bus = dev->bus->number;
374 	link->dev = PCI_SLOT(dev->devfn);
375 	atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
376 
377 	rc = alloc_spa(dev, link);
378 	if (rc)
379 		goto err_free;
380 
381 	rc = setup_xsl_irq(dev, link);
382 	if (rc)
383 		goto err_spa;
384 
385 	/* platform specific hook */
386 	rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
387 				&link->platform_data);
388 	if (rc)
389 		goto err_xsl_irq;
390 
391 	*out_link = link;
392 	return 0;
393 
394 err_xsl_irq:
395 	release_xsl_irq(link);
396 err_spa:
397 	free_spa(link);
398 err_free:
399 	kfree(link);
400 	return rc;
401 }
402 
free_link(struct link * link)403 static void free_link(struct link *link)
404 {
405 	release_xsl_irq(link);
406 	free_spa(link);
407 	kfree(link);
408 }
409 
ocxl_link_setup(struct pci_dev * dev,int PE_mask,void ** link_handle)410 int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
411 {
412 	int rc = 0;
413 	struct link *link;
414 
415 	mutex_lock(&links_list_lock);
416 	list_for_each_entry(link, &links_list, list) {
417 		/* The functions of a device all share the same link */
418 		if (link->domain == pci_domain_nr(dev->bus) &&
419 			link->bus == dev->bus->number &&
420 			link->dev == PCI_SLOT(dev->devfn)) {
421 			kref_get(&link->ref);
422 			*link_handle = link;
423 			goto unlock;
424 		}
425 	}
426 	rc = alloc_link(dev, PE_mask, &link);
427 	if (rc)
428 		goto unlock;
429 
430 	list_add(&link->list, &links_list);
431 	*link_handle = link;
432 unlock:
433 	mutex_unlock(&links_list_lock);
434 	return rc;
435 }
436 EXPORT_SYMBOL_GPL(ocxl_link_setup);
437 
release_xsl(struct kref * ref)438 static void release_xsl(struct kref *ref)
439 {
440 	struct link *link = container_of(ref, struct link, ref);
441 
442 	list_del(&link->list);
443 	/* call platform code before releasing data */
444 	pnv_ocxl_spa_release(link->platform_data);
445 	free_link(link);
446 }
447 
ocxl_link_release(struct pci_dev * dev,void * link_handle)448 void ocxl_link_release(struct pci_dev *dev, void *link_handle)
449 {
450 	struct link *link = (struct link *) link_handle;
451 
452 	mutex_lock(&links_list_lock);
453 	kref_put(&link->ref, release_xsl);
454 	mutex_unlock(&links_list_lock);
455 }
456 EXPORT_SYMBOL_GPL(ocxl_link_release);
457 
calculate_cfg_state(bool kernel)458 static u64 calculate_cfg_state(bool kernel)
459 {
460 	u64 state;
461 
462 	state = SPA_CFG_DR;
463 	if (mfspr(SPRN_LPCR) & LPCR_TC)
464 		state |= SPA_CFG_TC;
465 	if (radix_enabled())
466 		state |= SPA_CFG_XLAT_ror;
467 	else
468 		state |= SPA_CFG_XLAT_hpt;
469 	state |= SPA_CFG_HV;
470 	if (kernel) {
471 		if (mfmsr() & MSR_SF)
472 			state |= SPA_CFG_SF;
473 	} else {
474 		state |= SPA_CFG_PR;
475 		if (!test_tsk_thread_flag(current, TIF_32BIT))
476 			state |= SPA_CFG_SF;
477 	}
478 	return state;
479 }
480 
ocxl_link_add_pe(void * link_handle,int pasid,u32 pidr,u32 tidr,u64 amr,struct mm_struct * mm,void (* xsl_err_cb)(void * data,u64 addr,u64 dsisr),void * xsl_err_data)481 int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
482 		u64 amr, struct mm_struct *mm,
483 		void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
484 		void *xsl_err_data)
485 {
486 	struct link *link = (struct link *) link_handle;
487 	struct spa *spa = link->spa;
488 	struct ocxl_process_element *pe;
489 	int pe_handle, rc = 0;
490 	struct pe_data *pe_data;
491 
492 	BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
493 	if (pasid > SPA_PASID_MAX)
494 		return -EINVAL;
495 
496 	mutex_lock(&spa->spa_lock);
497 	pe_handle = pasid & SPA_PE_MASK;
498 	pe = spa->spa_mem + pe_handle;
499 
500 	if (pe->software_state) {
501 		rc = -EBUSY;
502 		goto unlock;
503 	}
504 
505 	pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
506 	if (!pe_data) {
507 		rc = -ENOMEM;
508 		goto unlock;
509 	}
510 
511 	pe_data->mm = mm;
512 	pe_data->xsl_err_cb = xsl_err_cb;
513 	pe_data->xsl_err_data = xsl_err_data;
514 
515 	memset(pe, 0, sizeof(struct ocxl_process_element));
516 	pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
517 	pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
518 	pe->pid = cpu_to_be32(pidr);
519 	pe->tid = cpu_to_be32(tidr);
520 	pe->amr = cpu_to_be64(amr);
521 	pe->software_state = cpu_to_be32(SPA_PE_VALID);
522 
523 	mm_context_add_copro(mm);
524 	/*
525 	 * Barrier is to make sure PE is visible in the SPA before it
526 	 * is used by the device. It also helps with the global TLBI
527 	 * invalidation
528 	 */
529 	mb();
530 	radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
531 
532 	/*
533 	 * The mm must stay valid for as long as the device uses it. We
534 	 * lower the count when the context is removed from the SPA.
535 	 *
536 	 * We grab mm_count (and not mm_users), as we don't want to
537 	 * end up in a circular dependency if a process mmaps its
538 	 * mmio, therefore incrementing the file ref count when
539 	 * calling mmap(), and forgets to unmap before exiting. In
540 	 * that scenario, when the kernel handles the death of the
541 	 * process, the file is not cleaned because unmap was not
542 	 * called, and the mm wouldn't be freed because we would still
543 	 * have a reference on mm_users. Incrementing mm_count solves
544 	 * the problem.
545 	 */
546 	mmgrab(mm);
547 	trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
548 unlock:
549 	mutex_unlock(&spa->spa_lock);
550 	return rc;
551 }
552 EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
553 
ocxl_link_update_pe(void * link_handle,int pasid,__u16 tid)554 int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
555 {
556 	struct link *link = (struct link *) link_handle;
557 	struct spa *spa = link->spa;
558 	struct ocxl_process_element *pe;
559 	int pe_handle, rc;
560 
561 	if (pasid > SPA_PASID_MAX)
562 		return -EINVAL;
563 
564 	pe_handle = pasid & SPA_PE_MASK;
565 	pe = spa->spa_mem + pe_handle;
566 
567 	mutex_lock(&spa->spa_lock);
568 
569 	pe->tid = cpu_to_be32(tid);
570 
571 	/*
572 	 * The barrier makes sure the PE is updated
573 	 * before we clear the NPU context cache below, so that the
574 	 * old PE cannot be reloaded erroneously.
575 	 */
576 	mb();
577 
578 	/*
579 	 * hook to platform code
580 	 * On powerpc, the entry needs to be cleared from the context
581 	 * cache of the NPU.
582 	 */
583 	rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
584 	WARN_ON(rc);
585 
586 	mutex_unlock(&spa->spa_lock);
587 	return rc;
588 }
589 
ocxl_link_remove_pe(void * link_handle,int pasid)590 int ocxl_link_remove_pe(void *link_handle, int pasid)
591 {
592 	struct link *link = (struct link *) link_handle;
593 	struct spa *spa = link->spa;
594 	struct ocxl_process_element *pe;
595 	struct pe_data *pe_data;
596 	int pe_handle, rc;
597 
598 	if (pasid > SPA_PASID_MAX)
599 		return -EINVAL;
600 
601 	/*
602 	 * About synchronization with our memory fault handler:
603 	 *
604 	 * Before removing the PE, the driver is supposed to have
605 	 * notified the AFU, which should have cleaned up and make
606 	 * sure the PASID is no longer in use, including pending
607 	 * interrupts. However, there's no way to be sure...
608 	 *
609 	 * We clear the PE and remove the context from our radix
610 	 * tree. From that point on, any new interrupt for that
611 	 * context will fail silently, which is ok. As mentioned
612 	 * above, that's not expected, but it could happen if the
613 	 * driver or AFU didn't do the right thing.
614 	 *
615 	 * There could still be a bottom half running, but we don't
616 	 * need to wait/flush, as it is managing a reference count on
617 	 * the mm it reads from the radix tree.
618 	 */
619 	pe_handle = pasid & SPA_PE_MASK;
620 	pe = spa->spa_mem + pe_handle;
621 
622 	mutex_lock(&spa->spa_lock);
623 
624 	if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
625 		rc = -EINVAL;
626 		goto unlock;
627 	}
628 
629 	trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
630 				be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
631 
632 	memset(pe, 0, sizeof(struct ocxl_process_element));
633 	/*
634 	 * The barrier makes sure the PE is removed from the SPA
635 	 * before we clear the NPU context cache below, so that the
636 	 * old PE cannot be reloaded erroneously.
637 	 */
638 	mb();
639 
640 	/*
641 	 * hook to platform code
642 	 * On powerpc, the entry needs to be cleared from the context
643 	 * cache of the NPU.
644 	 */
645 	rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
646 	WARN_ON(rc);
647 
648 	pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
649 	if (!pe_data) {
650 		WARN(1, "Couldn't find pe data when removing PE\n");
651 	} else {
652 		mm_context_remove_copro(pe_data->mm);
653 		mmdrop(pe_data->mm);
654 		kfree_rcu(pe_data, rcu);
655 	}
656 unlock:
657 	mutex_unlock(&spa->spa_lock);
658 	return rc;
659 }
660 EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
661 
ocxl_link_irq_alloc(void * link_handle,int * hw_irq,u64 * trigger_addr)662 int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr)
663 {
664 	struct link *link = (struct link *) link_handle;
665 	int rc, irq;
666 	u64 addr;
667 
668 	if (atomic_dec_if_positive(&link->irq_available) < 0)
669 		return -ENOSPC;
670 
671 	rc = pnv_ocxl_alloc_xive_irq(&irq, &addr);
672 	if (rc) {
673 		atomic_inc(&link->irq_available);
674 		return rc;
675 	}
676 
677 	*hw_irq = irq;
678 	*trigger_addr = addr;
679 	return 0;
680 }
681 EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
682 
ocxl_link_free_irq(void * link_handle,int hw_irq)683 void ocxl_link_free_irq(void *link_handle, int hw_irq)
684 {
685 	struct link *link = (struct link *) link_handle;
686 
687 	pnv_ocxl_free_xive_irq(hw_irq);
688 	atomic_inc(&link->irq_available);
689 }
690 EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
691