1 /*
2  * Copyright 2014 IBM Corp.
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public License
6  * as published by the Free Software Foundation; either version
7  * 2 of the License, or (at your option) any later version.
8  */
9 
10 #include <linux/interrupt.h>
11 #include <linux/workqueue.h>
12 #include <linux/sched.h>
13 #include <linux/wait.h>
14 #include <linux/slab.h>
15 #include <linux/pid.h>
16 #include <asm/cputable.h>
17 #include <misc/cxl-base.h>
18 
19 #include "cxl.h"
20 #include "trace.h"
21 
afu_irq_range_start(void)22 static int afu_irq_range_start(void)
23 {
24 	if (cpu_has_feature(CPU_FTR_HVMODE))
25 		return 1;
26 	return 0;
27 }
28 
schedule_cxl_fault(struct cxl_context * ctx,u64 dsisr,u64 dar)29 static irqreturn_t schedule_cxl_fault(struct cxl_context *ctx, u64 dsisr, u64 dar)
30 {
31 	ctx->dsisr = dsisr;
32 	ctx->dar = dar;
33 	schedule_work(&ctx->fault_work);
34 	return IRQ_HANDLED;
35 }
36 
cxl_irq_psl9(int irq,struct cxl_context * ctx,struct cxl_irq_info * irq_info)37 irqreturn_t cxl_irq_psl9(int irq, struct cxl_context *ctx, struct cxl_irq_info *irq_info)
38 {
39 	u64 dsisr, dar;
40 
41 	dsisr = irq_info->dsisr;
42 	dar = irq_info->dar;
43 
44 	trace_cxl_psl9_irq(ctx, irq, dsisr, dar);
45 
46 	pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
47 
48 	if (dsisr & CXL_PSL9_DSISR_An_TF) {
49 		pr_devel("CXL interrupt: Scheduling translation fault handling for later (pe: %i)\n", ctx->pe);
50 		return schedule_cxl_fault(ctx, dsisr, dar);
51 	}
52 
53 	if (dsisr & CXL_PSL9_DSISR_An_PE)
54 		return cxl_ops->handle_psl_slice_error(ctx, dsisr,
55 						irq_info->errstat);
56 	if (dsisr & CXL_PSL9_DSISR_An_AE) {
57 		pr_devel("CXL interrupt: AFU Error 0x%016llx\n", irq_info->afu_err);
58 
59 		if (ctx->pending_afu_err) {
60 			/*
61 			 * This shouldn't happen - the PSL treats these errors
62 			 * as fatal and will have reset the AFU, so there's not
63 			 * much point buffering multiple AFU errors.
64 			 * OTOH if we DO ever see a storm of these come in it's
65 			 * probably best that we log them somewhere:
66 			 */
67 			dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error undelivered to pe %i: 0x%016llx\n",
68 					    ctx->pe, irq_info->afu_err);
69 		} else {
70 			spin_lock(&ctx->lock);
71 			ctx->afu_err = irq_info->afu_err;
72 			ctx->pending_afu_err = 1;
73 			spin_unlock(&ctx->lock);
74 
75 			wake_up_all(&ctx->wq);
76 		}
77 
78 		cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
79 		return IRQ_HANDLED;
80 	}
81 	if (dsisr & CXL_PSL9_DSISR_An_OC)
82 		pr_devel("CXL interrupt: OS Context Warning\n");
83 
84 	WARN(1, "Unhandled CXL PSL IRQ\n");
85 	return IRQ_HANDLED;
86 }
87 
cxl_irq_psl8(int irq,struct cxl_context * ctx,struct cxl_irq_info * irq_info)88 irqreturn_t cxl_irq_psl8(int irq, struct cxl_context *ctx, struct cxl_irq_info *irq_info)
89 {
90 	u64 dsisr, dar;
91 
92 	dsisr = irq_info->dsisr;
93 	dar = irq_info->dar;
94 
95 	trace_cxl_psl_irq(ctx, irq, dsisr, dar);
96 
97 	pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
98 
99 	if (dsisr & CXL_PSL_DSISR_An_DS) {
100 		/*
101 		 * We don't inherently need to sleep to handle this, but we do
102 		 * need to get a ref to the task's mm, which we can't do from
103 		 * irq context without the potential for a deadlock since it
104 		 * takes the task_lock. An alternate option would be to keep a
105 		 * reference to the task's mm the entire time it has cxl open,
106 		 * but to do that we need to solve the issue where we hold a
107 		 * ref to the mm, but the mm can hold a ref to the fd after an
108 		 * mmap preventing anything from being cleaned up.
109 		 */
110 		pr_devel("Scheduling segment miss handling for later pe: %i\n", ctx->pe);
111 		return schedule_cxl_fault(ctx, dsisr, dar);
112 	}
113 
114 	if (dsisr & CXL_PSL_DSISR_An_M)
115 		pr_devel("CXL interrupt: PTE not found\n");
116 	if (dsisr & CXL_PSL_DSISR_An_P)
117 		pr_devel("CXL interrupt: Storage protection violation\n");
118 	if (dsisr & CXL_PSL_DSISR_An_A)
119 		pr_devel("CXL interrupt: AFU lock access to write through or cache inhibited storage\n");
120 	if (dsisr & CXL_PSL_DSISR_An_S)
121 		pr_devel("CXL interrupt: Access was afu_wr or afu_zero\n");
122 	if (dsisr & CXL_PSL_DSISR_An_K)
123 		pr_devel("CXL interrupt: Access not permitted by virtual page class key protection\n");
124 
125 	if (dsisr & CXL_PSL_DSISR_An_DM) {
126 		/*
127 		 * In some cases we might be able to handle the fault
128 		 * immediately if hash_page would succeed, but we still need
129 		 * the task's mm, which as above we can't get without a lock
130 		 */
131 		pr_devel("Scheduling page fault handling for later pe: %i\n", ctx->pe);
132 		return schedule_cxl_fault(ctx, dsisr, dar);
133 	}
134 	if (dsisr & CXL_PSL_DSISR_An_ST)
135 		WARN(1, "CXL interrupt: Segment Table PTE not found\n");
136 	if (dsisr & CXL_PSL_DSISR_An_UR)
137 		pr_devel("CXL interrupt: AURP PTE not found\n");
138 	if (dsisr & CXL_PSL_DSISR_An_PE)
139 		return cxl_ops->handle_psl_slice_error(ctx, dsisr,
140 						irq_info->errstat);
141 	if (dsisr & CXL_PSL_DSISR_An_AE) {
142 		pr_devel("CXL interrupt: AFU Error 0x%016llx\n", irq_info->afu_err);
143 
144 		if (ctx->pending_afu_err) {
145 			/*
146 			 * This shouldn't happen - the PSL treats these errors
147 			 * as fatal and will have reset the AFU, so there's not
148 			 * much point buffering multiple AFU errors.
149 			 * OTOH if we DO ever see a storm of these come in it's
150 			 * probably best that we log them somewhere:
151 			 */
152 			dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error "
153 					    "undelivered to pe %i: 0x%016llx\n",
154 					    ctx->pe, irq_info->afu_err);
155 		} else {
156 			spin_lock(&ctx->lock);
157 			ctx->afu_err = irq_info->afu_err;
158 			ctx->pending_afu_err = true;
159 			spin_unlock(&ctx->lock);
160 
161 			wake_up_all(&ctx->wq);
162 		}
163 
164 		cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
165 		return IRQ_HANDLED;
166 	}
167 	if (dsisr & CXL_PSL_DSISR_An_OC)
168 		pr_devel("CXL interrupt: OS Context Warning\n");
169 
170 	WARN(1, "Unhandled CXL PSL IRQ\n");
171 	return IRQ_HANDLED;
172 }
173 
cxl_irq_afu(int irq,void * data)174 static irqreturn_t cxl_irq_afu(int irq, void *data)
175 {
176 	struct cxl_context *ctx = data;
177 	irq_hw_number_t hwirq = irqd_to_hwirq(irq_get_irq_data(irq));
178 	int irq_off, afu_irq = 0;
179 	__u16 range;
180 	int r;
181 
182 	/*
183 	 * Look for the interrupt number.
184 	 * On bare-metal, we know range 0 only contains the PSL
185 	 * interrupt so we could start counting at range 1 and initialize
186 	 * afu_irq at 1.
187 	 * In a guest, range 0 also contains AFU interrupts, so it must
188 	 * be counted for. Therefore we initialize afu_irq at 0 to take into
189 	 * account the PSL interrupt.
190 	 *
191 	 * For code-readability, it just seems easier to go over all
192 	 * the ranges on bare-metal and guest. The end result is the same.
193 	 */
194 	for (r = 0; r < CXL_IRQ_RANGES; r++) {
195 		irq_off = hwirq - ctx->irqs.offset[r];
196 		range = ctx->irqs.range[r];
197 		if (irq_off >= 0 && irq_off < range) {
198 			afu_irq += irq_off;
199 			break;
200 		}
201 		afu_irq += range;
202 	}
203 	if (unlikely(r >= CXL_IRQ_RANGES)) {
204 		WARN(1, "Received AFU IRQ out of range for pe %i (virq %i hwirq %lx)\n",
205 		     ctx->pe, irq, hwirq);
206 		return IRQ_HANDLED;
207 	}
208 
209 	trace_cxl_afu_irq(ctx, afu_irq, irq, hwirq);
210 	pr_devel("Received AFU interrupt %i for pe: %i (virq %i hwirq %lx)\n",
211 	       afu_irq, ctx->pe, irq, hwirq);
212 
213 	if (unlikely(!ctx->irq_bitmap)) {
214 		WARN(1, "Received AFU IRQ for context with no IRQ bitmap\n");
215 		return IRQ_HANDLED;
216 	}
217 	spin_lock(&ctx->lock);
218 	set_bit(afu_irq - 1, ctx->irq_bitmap);
219 	ctx->pending_irq = true;
220 	spin_unlock(&ctx->lock);
221 
222 	wake_up_all(&ctx->wq);
223 
224 	return IRQ_HANDLED;
225 }
226 
cxl_map_irq(struct cxl * adapter,irq_hw_number_t hwirq,irq_handler_t handler,void * cookie,const char * name)227 unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
228 			 irq_handler_t handler, void *cookie, const char *name)
229 {
230 	unsigned int virq;
231 	int result;
232 
233 	/* IRQ Domain? */
234 	virq = irq_create_mapping(NULL, hwirq);
235 	if (!virq) {
236 		dev_warn(&adapter->dev, "cxl_map_irq: irq_create_mapping failed\n");
237 		return 0;
238 	}
239 
240 	if (cxl_ops->setup_irq)
241 		cxl_ops->setup_irq(adapter, hwirq, virq);
242 
243 	pr_devel("hwirq %#lx mapped to virq %u\n", hwirq, virq);
244 
245 	result = request_irq(virq, handler, 0, name, cookie);
246 	if (result) {
247 		dev_warn(&adapter->dev, "cxl_map_irq: request_irq failed: %i\n", result);
248 		return 0;
249 	}
250 
251 	return virq;
252 }
253 
cxl_unmap_irq(unsigned int virq,void * cookie)254 void cxl_unmap_irq(unsigned int virq, void *cookie)
255 {
256 	free_irq(virq, cookie);
257 }
258 
cxl_register_one_irq(struct cxl * adapter,irq_handler_t handler,void * cookie,irq_hw_number_t * dest_hwirq,unsigned int * dest_virq,const char * name)259 int cxl_register_one_irq(struct cxl *adapter,
260 			irq_handler_t handler,
261 			void *cookie,
262 			irq_hw_number_t *dest_hwirq,
263 			unsigned int *dest_virq,
264 			const char *name)
265 {
266 	int hwirq, virq;
267 
268 	if ((hwirq = cxl_ops->alloc_one_irq(adapter)) < 0)
269 		return hwirq;
270 
271 	if (!(virq = cxl_map_irq(adapter, hwirq, handler, cookie, name)))
272 		goto err;
273 
274 	*dest_hwirq = hwirq;
275 	*dest_virq = virq;
276 
277 	return 0;
278 
279 err:
280 	cxl_ops->release_one_irq(adapter, hwirq);
281 	return -ENOMEM;
282 }
283 
afu_irq_name_free(struct cxl_context * ctx)284 void afu_irq_name_free(struct cxl_context *ctx)
285 {
286 	struct cxl_irq_name *irq_name, *tmp;
287 
288 	list_for_each_entry_safe(irq_name, tmp, &ctx->irq_names, list) {
289 		kfree(irq_name->name);
290 		list_del(&irq_name->list);
291 		kfree(irq_name);
292 	}
293 }
294 
afu_allocate_irqs(struct cxl_context * ctx,u32 count)295 int afu_allocate_irqs(struct cxl_context *ctx, u32 count)
296 {
297 	int rc, r, i, j = 1;
298 	struct cxl_irq_name *irq_name;
299 	int alloc_count;
300 
301 	/*
302 	 * In native mode, range 0 is reserved for the multiplexed
303 	 * PSL interrupt. It has been allocated when the AFU was initialized.
304 	 *
305 	 * In a guest, the PSL interrupt is not mutliplexed, but per-context,
306 	 * and is the first interrupt from range 0. It still needs to be
307 	 * allocated, so bump the count by one.
308 	 */
309 	if (cpu_has_feature(CPU_FTR_HVMODE))
310 		alloc_count = count;
311 	else
312 		alloc_count = count + 1;
313 
314 	if ((rc = cxl_ops->alloc_irq_ranges(&ctx->irqs, ctx->afu->adapter,
315 							alloc_count)))
316 		return rc;
317 
318 	if (cpu_has_feature(CPU_FTR_HVMODE)) {
319 		/* Multiplexed PSL Interrupt */
320 		ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
321 		ctx->irqs.range[0] = 1;
322 	}
323 
324 	ctx->irq_count = count;
325 	ctx->irq_bitmap = kcalloc(BITS_TO_LONGS(count),
326 				  sizeof(*ctx->irq_bitmap), GFP_KERNEL);
327 	if (!ctx->irq_bitmap)
328 		goto out;
329 
330 	/*
331 	 * Allocate names first.  If any fail, bail out before allocating
332 	 * actual hardware IRQs.
333 	 */
334 	for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
335 		for (i = 0; i < ctx->irqs.range[r]; i++) {
336 			irq_name = kmalloc(sizeof(struct cxl_irq_name),
337 					   GFP_KERNEL);
338 			if (!irq_name)
339 				goto out;
340 			irq_name->name = kasprintf(GFP_KERNEL, "cxl-%s-pe%i-%i",
341 						   dev_name(&ctx->afu->dev),
342 						   ctx->pe, j);
343 			if (!irq_name->name) {
344 				kfree(irq_name);
345 				goto out;
346 			}
347 			/* Add to tail so next look get the correct order */
348 			list_add_tail(&irq_name->list, &ctx->irq_names);
349 			j++;
350 		}
351 	}
352 	return 0;
353 
354 out:
355 	cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
356 	bitmap_free(ctx->irq_bitmap);
357 	afu_irq_name_free(ctx);
358 	return -ENOMEM;
359 }
360 
afu_register_hwirqs(struct cxl_context * ctx)361 static void afu_register_hwirqs(struct cxl_context *ctx)
362 {
363 	irq_hw_number_t hwirq;
364 	struct cxl_irq_name *irq_name;
365 	int r, i;
366 	irqreturn_t (*handler)(int irq, void *data);
367 
368 	/* We've allocated all memory now, so let's do the irq allocations */
369 	irq_name = list_first_entry(&ctx->irq_names, struct cxl_irq_name, list);
370 	for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
371 		hwirq = ctx->irqs.offset[r];
372 		for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
373 			if (r == 0 && i == 0)
374 				/*
375 				 * The very first interrupt of range 0 is
376 				 * always the PSL interrupt, but we only
377 				 * need to connect a handler for guests,
378 				 * because there's one PSL interrupt per
379 				 * context.
380 				 * On bare-metal, the PSL interrupt is
381 				 * multiplexed and was setup when the AFU
382 				 * was configured.
383 				 */
384 				handler = cxl_ops->psl_interrupt;
385 			else
386 				handler = cxl_irq_afu;
387 			cxl_map_irq(ctx->afu->adapter, hwirq, handler, ctx,
388 				irq_name->name);
389 			irq_name = list_next_entry(irq_name, list);
390 		}
391 	}
392 }
393 
afu_register_irqs(struct cxl_context * ctx,u32 count)394 int afu_register_irqs(struct cxl_context *ctx, u32 count)
395 {
396 	int rc;
397 
398 	rc = afu_allocate_irqs(ctx, count);
399 	if (rc)
400 		return rc;
401 
402 	afu_register_hwirqs(ctx);
403 	return 0;
404 }
405 
afu_release_irqs(struct cxl_context * ctx,void * cookie)406 void afu_release_irqs(struct cxl_context *ctx, void *cookie)
407 {
408 	irq_hw_number_t hwirq;
409 	unsigned int virq;
410 	int r, i;
411 
412 	for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
413 		hwirq = ctx->irqs.offset[r];
414 		for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
415 			virq = irq_find_mapping(NULL, hwirq);
416 			if (virq)
417 				cxl_unmap_irq(virq, cookie);
418 		}
419 	}
420 
421 	afu_irq_name_free(ctx);
422 	cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
423 
424 	ctx->irq_count = 0;
425 }
426 
cxl_afu_decode_psl_serr(struct cxl_afu * afu,u64 serr)427 void cxl_afu_decode_psl_serr(struct cxl_afu *afu, u64 serr)
428 {
429 	dev_crit(&afu->dev,
430 		 "PSL Slice error received. Check AFU for root cause.\n");
431 	dev_crit(&afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
432 	if (serr & CXL_PSL_SERR_An_afuto)
433 		dev_crit(&afu->dev, "AFU MMIO Timeout\n");
434 	if (serr & CXL_PSL_SERR_An_afudis)
435 		dev_crit(&afu->dev,
436 			 "MMIO targeted Accelerator that was not enabled\n");
437 	if (serr & CXL_PSL_SERR_An_afuov)
438 		dev_crit(&afu->dev, "AFU CTAG Overflow\n");
439 	if (serr & CXL_PSL_SERR_An_badsrc)
440 		dev_crit(&afu->dev, "Bad Interrupt Source\n");
441 	if (serr & CXL_PSL_SERR_An_badctx)
442 		dev_crit(&afu->dev, "Bad Context Handle\n");
443 	if (serr & CXL_PSL_SERR_An_llcmdis)
444 		dev_crit(&afu->dev, "LLCMD to Disabled AFU\n");
445 	if (serr & CXL_PSL_SERR_An_llcmdto)
446 		dev_crit(&afu->dev, "LLCMD Timeout to AFU\n");
447 	if (serr & CXL_PSL_SERR_An_afupar)
448 		dev_crit(&afu->dev, "AFU MMIO Parity Error\n");
449 	if (serr & CXL_PSL_SERR_An_afudup)
450 		dev_crit(&afu->dev, "AFU MMIO Duplicate CTAG Error\n");
451 	if (serr & CXL_PSL_SERR_An_AE)
452 		dev_crit(&afu->dev,
453 			 "AFU asserted JDONE with JERROR in AFU Directed Mode\n");
454 }
455