1 /* Virtio ring implementation.
2  *
3  *  Copyright 2007 Rusty Russell IBM Corporation
4  *
5  *  This program is free software; you can redistribute it and/or modify
6  *  it under the terms of the GNU General Public License as published by
7  *  the Free Software Foundation; either version 2 of the License, or
8  *  (at your option) any later version.
9  *
10  *  This program is distributed in the hope that it will be useful,
11  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  *  GNU General Public License for more details.
14  *
15  *  You should have received a copy of the GNU General Public License
16  *  along with this program; if not, write to the Free Software
17  *  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
18  */
19 #include <linux/virtio.h>
20 #include <linux/virtio_ring.h>
21 #include <linux/virtio_config.h>
22 #include <linux/device.h>
23 #include <linux/slab.h>
24 #include <linux/module.h>
25 #include <linux/hrtimer.h>
26 #include <linux/dma-mapping.h>
27 #include <xen/xen.h>
28 
29 #ifdef DEBUG
30 /* For development, we want to crash whenever the ring is screwed. */
31 #define BAD_RING(_vq, fmt, args...)				\
32 	do {							\
33 		dev_err(&(_vq)->vq.vdev->dev,			\
34 			"%s:"fmt, (_vq)->vq.name, ##args);	\
35 		BUG();						\
36 	} while (0)
37 /* Caller is supposed to guarantee no reentry. */
38 #define START_USE(_vq)						\
39 	do {							\
40 		if ((_vq)->in_use)				\
41 			panic("%s:in_use = %i\n",		\
42 			      (_vq)->vq.name, (_vq)->in_use);	\
43 		(_vq)->in_use = __LINE__;			\
44 	} while (0)
45 #define END_USE(_vq) \
46 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
47 #else
48 #define BAD_RING(_vq, fmt, args...)				\
49 	do {							\
50 		dev_err(&_vq->vq.vdev->dev,			\
51 			"%s:"fmt, (_vq)->vq.name, ##args);	\
52 		(_vq)->broken = true;				\
53 	} while (0)
54 #define START_USE(vq)
55 #define END_USE(vq)
56 #endif
57 
58 struct vring_desc_state {
59 	void *data;			/* Data for callback. */
60 	struct vring_desc *indir_desc;	/* Indirect descriptor, if any. */
61 };
62 
63 struct vring_virtqueue {
64 	struct virtqueue vq;
65 
66 	/* Actual memory layout for this queue */
67 	struct vring vring;
68 
69 	/* Can we use weak barriers? */
70 	bool weak_barriers;
71 
72 	/* Other side has made a mess, don't try any more. */
73 	bool broken;
74 
75 	/* Host supports indirect buffers */
76 	bool indirect;
77 
78 	/* Host publishes avail event idx */
79 	bool event;
80 
81 	/* Head of free buffer list. */
82 	unsigned int free_head;
83 	/* Number we've added since last sync. */
84 	unsigned int num_added;
85 
86 	/* Last used index we've seen. */
87 	u16 last_used_idx;
88 
89 	/* Last written value to avail->flags */
90 	u16 avail_flags_shadow;
91 
92 	/* Last written value to avail->idx in guest byte order */
93 	u16 avail_idx_shadow;
94 
95 	/* How to notify other side. FIXME: commonalize hcalls! */
96 	bool (*notify)(struct virtqueue *vq);
97 
98 	/* DMA, allocation, and size information */
99 	bool we_own_ring;
100 	size_t queue_size_in_bytes;
101 	dma_addr_t queue_dma_addr;
102 
103 #ifdef DEBUG
104 	/* They're supposed to lock for us. */
105 	unsigned int in_use;
106 
107 	/* Figure out if their kicks are too delayed. */
108 	bool last_add_time_valid;
109 	ktime_t last_add_time;
110 #endif
111 
112 	/* Per-descriptor state. */
113 	struct vring_desc_state desc_state[];
114 };
115 
116 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
117 
118 /*
119  * Modern virtio devices have feature bits to specify whether they need a
120  * quirk and bypass the IOMMU. If not there, just use the DMA API.
121  *
122  * If there, the interaction between virtio and DMA API is messy.
123  *
124  * On most systems with virtio, physical addresses match bus addresses,
125  * and it doesn't particularly matter whether we use the DMA API.
126  *
127  * On some systems, including Xen and any system with a physical device
128  * that speaks virtio behind a physical IOMMU, we must use the DMA API
129  * for virtio DMA to work at all.
130  *
131  * On other systems, including SPARC and PPC64, virtio-pci devices are
132  * enumerated as though they are behind an IOMMU, but the virtio host
133  * ignores the IOMMU, so we must either pretend that the IOMMU isn't
134  * there or somehow map everything as the identity.
135  *
136  * For the time being, we preserve historic behavior and bypass the DMA
137  * API.
138  *
139  * TODO: install a per-device DMA ops structure that does the right thing
140  * taking into account all the above quirks, and use the DMA API
141  * unconditionally on data path.
142  */
143 
vring_use_dma_api(struct virtio_device * vdev)144 static bool vring_use_dma_api(struct virtio_device *vdev)
145 {
146 	if (!virtio_has_iommu_quirk(vdev))
147 		return true;
148 
149 	/* Otherwise, we are left to guess. */
150 	/*
151 	 * In theory, it's possible to have a buggy QEMU-supposed
152 	 * emulated Q35 IOMMU and Xen enabled at the same time.  On
153 	 * such a configuration, virtio has never worked and will
154 	 * not work without an even larger kludge.  Instead, enable
155 	 * the DMA API if we're a Xen guest, which at least allows
156 	 * all of the sensible Xen configurations to work correctly.
157 	 */
158 	if (xen_domain())
159 		return true;
160 
161 	return false;
162 }
163 
164 /*
165  * The DMA ops on various arches are rather gnarly right now, and
166  * making all of the arch DMA ops work on the vring device itself
167  * is a mess.  For now, we use the parent device for DMA ops.
168  */
vring_dma_dev(const struct vring_virtqueue * vq)169 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
170 {
171 	return vq->vq.vdev->dev.parent;
172 }
173 
174 /* Map one sg entry. */
vring_map_one_sg(const struct vring_virtqueue * vq,struct scatterlist * sg,enum dma_data_direction direction)175 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
176 				   struct scatterlist *sg,
177 				   enum dma_data_direction direction)
178 {
179 	if (!vring_use_dma_api(vq->vq.vdev))
180 		return (dma_addr_t)sg_phys(sg);
181 
182 	/*
183 	 * We can't use dma_map_sg, because we don't use scatterlists in
184 	 * the way it expects (we don't guarantee that the scatterlist
185 	 * will exist for the lifetime of the mapping).
186 	 */
187 	return dma_map_page(vring_dma_dev(vq),
188 			    sg_page(sg), sg->offset, sg->length,
189 			    direction);
190 }
191 
vring_map_single(const struct vring_virtqueue * vq,void * cpu_addr,size_t size,enum dma_data_direction direction)192 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
193 				   void *cpu_addr, size_t size,
194 				   enum dma_data_direction direction)
195 {
196 	if (!vring_use_dma_api(vq->vq.vdev))
197 		return (dma_addr_t)virt_to_phys(cpu_addr);
198 
199 	return dma_map_single(vring_dma_dev(vq),
200 			      cpu_addr, size, direction);
201 }
202 
vring_unmap_one(const struct vring_virtqueue * vq,struct vring_desc * desc)203 static void vring_unmap_one(const struct vring_virtqueue *vq,
204 			    struct vring_desc *desc)
205 {
206 	u16 flags;
207 
208 	if (!vring_use_dma_api(vq->vq.vdev))
209 		return;
210 
211 	flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
212 
213 	if (flags & VRING_DESC_F_INDIRECT) {
214 		dma_unmap_single(vring_dma_dev(vq),
215 				 virtio64_to_cpu(vq->vq.vdev, desc->addr),
216 				 virtio32_to_cpu(vq->vq.vdev, desc->len),
217 				 (flags & VRING_DESC_F_WRITE) ?
218 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
219 	} else {
220 		dma_unmap_page(vring_dma_dev(vq),
221 			       virtio64_to_cpu(vq->vq.vdev, desc->addr),
222 			       virtio32_to_cpu(vq->vq.vdev, desc->len),
223 			       (flags & VRING_DESC_F_WRITE) ?
224 			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
225 	}
226 }
227 
vring_mapping_error(const struct vring_virtqueue * vq,dma_addr_t addr)228 static int vring_mapping_error(const struct vring_virtqueue *vq,
229 			       dma_addr_t addr)
230 {
231 	if (!vring_use_dma_api(vq->vq.vdev))
232 		return 0;
233 
234 	return dma_mapping_error(vring_dma_dev(vq), addr);
235 }
236 
alloc_indirect(struct virtqueue * _vq,unsigned int total_sg,gfp_t gfp)237 static struct vring_desc *alloc_indirect(struct virtqueue *_vq,
238 					 unsigned int total_sg, gfp_t gfp)
239 {
240 	struct vring_desc *desc;
241 	unsigned int i;
242 
243 	/*
244 	 * We require lowmem mappings for the descriptors because
245 	 * otherwise virt_to_phys will give us bogus addresses in the
246 	 * virtqueue.
247 	 */
248 	gfp &= ~__GFP_HIGHMEM;
249 
250 	desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
251 	if (!desc)
252 		return NULL;
253 
254 	for (i = 0; i < total_sg; i++)
255 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
256 	return desc;
257 }
258 
virtqueue_add(struct virtqueue * _vq,struct scatterlist * sgs[],unsigned int total_sg,unsigned int out_sgs,unsigned int in_sgs,void * data,void * ctx,gfp_t gfp)259 static inline int virtqueue_add(struct virtqueue *_vq,
260 				struct scatterlist *sgs[],
261 				unsigned int total_sg,
262 				unsigned int out_sgs,
263 				unsigned int in_sgs,
264 				void *data,
265 				void *ctx,
266 				gfp_t gfp)
267 {
268 	struct vring_virtqueue *vq = to_vvq(_vq);
269 	struct scatterlist *sg;
270 	struct vring_desc *desc;
271 	unsigned int i, n, avail, descs_used, prev, err_idx;
272 	int head;
273 	bool indirect;
274 
275 	START_USE(vq);
276 
277 	BUG_ON(data == NULL);
278 	BUG_ON(ctx && vq->indirect);
279 
280 	if (unlikely(vq->broken)) {
281 		END_USE(vq);
282 		return -EIO;
283 	}
284 
285 #ifdef DEBUG
286 	{
287 		ktime_t now = ktime_get();
288 
289 		/* No kick or get, with .1 second between?  Warn. */
290 		if (vq->last_add_time_valid)
291 			WARN_ON(ktime_to_ms(ktime_sub(now, vq->last_add_time))
292 					    > 100);
293 		vq->last_add_time = now;
294 		vq->last_add_time_valid = true;
295 	}
296 #endif
297 
298 	BUG_ON(total_sg == 0);
299 
300 	head = vq->free_head;
301 
302 	/* If the host supports indirect descriptor tables, and we have multiple
303 	 * buffers, then go indirect. FIXME: tune this threshold */
304 	if (vq->indirect && total_sg > 1 && vq->vq.num_free)
305 		desc = alloc_indirect(_vq, total_sg, gfp);
306 	else {
307 		desc = NULL;
308 		WARN_ON_ONCE(total_sg > vq->vring.num && !vq->indirect);
309 	}
310 
311 	if (desc) {
312 		/* Use a single buffer which doesn't continue */
313 		indirect = true;
314 		/* Set up rest to use this indirect table. */
315 		i = 0;
316 		descs_used = 1;
317 	} else {
318 		indirect = false;
319 		desc = vq->vring.desc;
320 		i = head;
321 		descs_used = total_sg;
322 	}
323 
324 	if (vq->vq.num_free < descs_used) {
325 		pr_debug("Can't add buf len %i - avail = %i\n",
326 			 descs_used, vq->vq.num_free);
327 		/* FIXME: for historical reasons, we force a notify here if
328 		 * there are outgoing parts to the buffer.  Presumably the
329 		 * host should service the ring ASAP. */
330 		if (out_sgs)
331 			vq->notify(&vq->vq);
332 		if (indirect)
333 			kfree(desc);
334 		END_USE(vq);
335 		return -ENOSPC;
336 	}
337 
338 	for (n = 0; n < out_sgs; n++) {
339 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
340 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
341 			if (vring_mapping_error(vq, addr))
342 				goto unmap_release;
343 
344 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT);
345 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
346 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
347 			prev = i;
348 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);
349 		}
350 	}
351 	for (; n < (out_sgs + in_sgs); n++) {
352 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
353 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
354 			if (vring_mapping_error(vq, addr))
355 				goto unmap_release;
356 
357 			desc[i].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE);
358 			desc[i].addr = cpu_to_virtio64(_vq->vdev, addr);
359 			desc[i].len = cpu_to_virtio32(_vq->vdev, sg->length);
360 			prev = i;
361 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);
362 		}
363 	}
364 	/* Last one doesn't continue. */
365 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
366 
367 	if (indirect) {
368 		/* Now that the indirect table is filled in, map it. */
369 		dma_addr_t addr = vring_map_single(
370 			vq, desc, total_sg * sizeof(struct vring_desc),
371 			DMA_TO_DEVICE);
372 		if (vring_mapping_error(vq, addr))
373 			goto unmap_release;
374 
375 		vq->vring.desc[head].flags = cpu_to_virtio16(_vq->vdev, VRING_DESC_F_INDIRECT);
376 		vq->vring.desc[head].addr = cpu_to_virtio64(_vq->vdev, addr);
377 
378 		vq->vring.desc[head].len = cpu_to_virtio32(_vq->vdev, total_sg * sizeof(struct vring_desc));
379 	}
380 
381 	/* We're using some buffers from the free list. */
382 	vq->vq.num_free -= descs_used;
383 
384 	/* Update free pointer */
385 	if (indirect)
386 		vq->free_head = virtio16_to_cpu(_vq->vdev, vq->vring.desc[head].next);
387 	else
388 		vq->free_head = i;
389 
390 	/* Store token and indirect buffer state. */
391 	vq->desc_state[head].data = data;
392 	if (indirect)
393 		vq->desc_state[head].indir_desc = desc;
394 	else
395 		vq->desc_state[head].indir_desc = ctx;
396 
397 	/* Put entry in available array (but don't update avail->idx until they
398 	 * do sync). */
399 	avail = vq->avail_idx_shadow & (vq->vring.num - 1);
400 	vq->vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
401 
402 	/* Descriptors and available array need to be set before we expose the
403 	 * new available array entries. */
404 	virtio_wmb(vq->weak_barriers);
405 	vq->avail_idx_shadow++;
406 	vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);
407 	vq->num_added++;
408 
409 	pr_debug("Added buffer head %i to %p\n", head, vq);
410 	END_USE(vq);
411 
412 	/* This is very unlikely, but theoretically possible.  Kick
413 	 * just in case. */
414 	if (unlikely(vq->num_added == (1 << 16) - 1))
415 		virtqueue_kick(_vq);
416 
417 	return 0;
418 
419 unmap_release:
420 	err_idx = i;
421 	i = head;
422 
423 	for (n = 0; n < total_sg; n++) {
424 		if (i == err_idx)
425 			break;
426 		vring_unmap_one(vq, &desc[i]);
427 		i = virtio16_to_cpu(_vq->vdev, vq->vring.desc[i].next);
428 	}
429 
430 	if (indirect)
431 		kfree(desc);
432 
433 	END_USE(vq);
434 	return -ENOMEM;
435 }
436 
437 /**
438  * virtqueue_add_sgs - expose buffers to other end
439  * @vq: the struct virtqueue we're talking about.
440  * @sgs: array of terminated scatterlists.
441  * @out_num: the number of scatterlists readable by other side
442  * @in_num: the number of scatterlists which are writable (after readable ones)
443  * @data: the token identifying the buffer.
444  * @gfp: how to do memory allocations (if necessary).
445  *
446  * Caller must ensure we don't call this with other virtqueue operations
447  * at the same time (except where noted).
448  *
449  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
450  */
virtqueue_add_sgs(struct virtqueue * _vq,struct scatterlist * sgs[],unsigned int out_sgs,unsigned int in_sgs,void * data,gfp_t gfp)451 int virtqueue_add_sgs(struct virtqueue *_vq,
452 		      struct scatterlist *sgs[],
453 		      unsigned int out_sgs,
454 		      unsigned int in_sgs,
455 		      void *data,
456 		      gfp_t gfp)
457 {
458 	unsigned int i, total_sg = 0;
459 
460 	/* Count them first. */
461 	for (i = 0; i < out_sgs + in_sgs; i++) {
462 		struct scatterlist *sg;
463 		for (sg = sgs[i]; sg; sg = sg_next(sg))
464 			total_sg++;
465 	}
466 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
467 			     data, NULL, gfp);
468 }
469 EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
470 
471 /**
472  * virtqueue_add_outbuf - expose output buffers to other end
473  * @vq: the struct virtqueue we're talking about.
474  * @sg: scatterlist (must be well-formed and terminated!)
475  * @num: the number of entries in @sg readable by other side
476  * @data: the token identifying the buffer.
477  * @gfp: how to do memory allocations (if necessary).
478  *
479  * Caller must ensure we don't call this with other virtqueue operations
480  * at the same time (except where noted).
481  *
482  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
483  */
virtqueue_add_outbuf(struct virtqueue * vq,struct scatterlist * sg,unsigned int num,void * data,gfp_t gfp)484 int virtqueue_add_outbuf(struct virtqueue *vq,
485 			 struct scatterlist *sg, unsigned int num,
486 			 void *data,
487 			 gfp_t gfp)
488 {
489 	return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
490 }
491 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
492 
493 /**
494  * virtqueue_add_inbuf - expose input buffers to other end
495  * @vq: the struct virtqueue we're talking about.
496  * @sg: scatterlist (must be well-formed and terminated!)
497  * @num: the number of entries in @sg writable by other side
498  * @data: the token identifying the buffer.
499  * @gfp: how to do memory allocations (if necessary).
500  *
501  * Caller must ensure we don't call this with other virtqueue operations
502  * at the same time (except where noted).
503  *
504  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
505  */
virtqueue_add_inbuf(struct virtqueue * vq,struct scatterlist * sg,unsigned int num,void * data,gfp_t gfp)506 int virtqueue_add_inbuf(struct virtqueue *vq,
507 			struct scatterlist *sg, unsigned int num,
508 			void *data,
509 			gfp_t gfp)
510 {
511 	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
512 }
513 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
514 
515 /**
516  * virtqueue_add_inbuf_ctx - expose input buffers to other end
517  * @vq: the struct virtqueue we're talking about.
518  * @sg: scatterlist (must be well-formed and terminated!)
519  * @num: the number of entries in @sg writable by other side
520  * @data: the token identifying the buffer.
521  * @ctx: extra context for the token
522  * @gfp: how to do memory allocations (if necessary).
523  *
524  * Caller must ensure we don't call this with other virtqueue operations
525  * at the same time (except where noted).
526  *
527  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
528  */
virtqueue_add_inbuf_ctx(struct virtqueue * vq,struct scatterlist * sg,unsigned int num,void * data,void * ctx,gfp_t gfp)529 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
530 			struct scatterlist *sg, unsigned int num,
531 			void *data,
532 			void *ctx,
533 			gfp_t gfp)
534 {
535 	return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
536 }
537 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
538 
539 /**
540  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
541  * @vq: the struct virtqueue
542  *
543  * Instead of virtqueue_kick(), you can do:
544  *	if (virtqueue_kick_prepare(vq))
545  *		virtqueue_notify(vq);
546  *
547  * This is sometimes useful because the virtqueue_kick_prepare() needs
548  * to be serialized, but the actual virtqueue_notify() call does not.
549  */
virtqueue_kick_prepare(struct virtqueue * _vq)550 bool virtqueue_kick_prepare(struct virtqueue *_vq)
551 {
552 	struct vring_virtqueue *vq = to_vvq(_vq);
553 	u16 new, old;
554 	bool needs_kick;
555 
556 	START_USE(vq);
557 	/* We need to expose available array entries before checking avail
558 	 * event. */
559 	virtio_mb(vq->weak_barriers);
560 
561 	old = vq->avail_idx_shadow - vq->num_added;
562 	new = vq->avail_idx_shadow;
563 	vq->num_added = 0;
564 
565 #ifdef DEBUG
566 	if (vq->last_add_time_valid) {
567 		WARN_ON(ktime_to_ms(ktime_sub(ktime_get(),
568 					      vq->last_add_time)) > 100);
569 	}
570 	vq->last_add_time_valid = false;
571 #endif
572 
573 	if (vq->event) {
574 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, vring_avail_event(&vq->vring)),
575 					      new, old);
576 	} else {
577 		needs_kick = !(vq->vring.used->flags & cpu_to_virtio16(_vq->vdev, VRING_USED_F_NO_NOTIFY));
578 	}
579 	END_USE(vq);
580 	return needs_kick;
581 }
582 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
583 
584 /**
585  * virtqueue_notify - second half of split virtqueue_kick call.
586  * @vq: the struct virtqueue
587  *
588  * This does not need to be serialized.
589  *
590  * Returns false if host notify failed or queue is broken, otherwise true.
591  */
virtqueue_notify(struct virtqueue * _vq)592 bool virtqueue_notify(struct virtqueue *_vq)
593 {
594 	struct vring_virtqueue *vq = to_vvq(_vq);
595 
596 	if (unlikely(vq->broken))
597 		return false;
598 
599 	/* Prod other side to tell it about changes. */
600 	if (!vq->notify(_vq)) {
601 		vq->broken = true;
602 		return false;
603 	}
604 	return true;
605 }
606 EXPORT_SYMBOL_GPL(virtqueue_notify);
607 
608 /**
609  * virtqueue_kick - update after add_buf
610  * @vq: the struct virtqueue
611  *
612  * After one or more virtqueue_add_* calls, invoke this to kick
613  * the other side.
614  *
615  * Caller must ensure we don't call this with other virtqueue
616  * operations at the same time (except where noted).
617  *
618  * Returns false if kick failed, otherwise true.
619  */
virtqueue_kick(struct virtqueue * vq)620 bool virtqueue_kick(struct virtqueue *vq)
621 {
622 	if (virtqueue_kick_prepare(vq))
623 		return virtqueue_notify(vq);
624 	return true;
625 }
626 EXPORT_SYMBOL_GPL(virtqueue_kick);
627 
detach_buf(struct vring_virtqueue * vq,unsigned int head,void ** ctx)628 static void detach_buf(struct vring_virtqueue *vq, unsigned int head,
629 		       void **ctx)
630 {
631 	unsigned int i, j;
632 	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
633 
634 	/* Clear data ptr. */
635 	vq->desc_state[head].data = NULL;
636 
637 	/* Put back on free list: unmap first-level descriptors and find end */
638 	i = head;
639 
640 	while (vq->vring.desc[i].flags & nextflag) {
641 		vring_unmap_one(vq, &vq->vring.desc[i]);
642 		i = virtio16_to_cpu(vq->vq.vdev, vq->vring.desc[i].next);
643 		vq->vq.num_free++;
644 	}
645 
646 	vring_unmap_one(vq, &vq->vring.desc[i]);
647 	vq->vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev, vq->free_head);
648 	vq->free_head = head;
649 
650 	/* Plus final descriptor */
651 	vq->vq.num_free++;
652 
653 	if (vq->indirect) {
654 		struct vring_desc *indir_desc = vq->desc_state[head].indir_desc;
655 		u32 len;
656 
657 		/* Free the indirect table, if any, now that it's unmapped. */
658 		if (!indir_desc)
659 			return;
660 
661 		len = virtio32_to_cpu(vq->vq.vdev, vq->vring.desc[head].len);
662 
663 		BUG_ON(!(vq->vring.desc[head].flags &
664 			 cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_INDIRECT)));
665 		BUG_ON(len == 0 || len % sizeof(struct vring_desc));
666 
667 		for (j = 0; j < len / sizeof(struct vring_desc); j++)
668 			vring_unmap_one(vq, &indir_desc[j]);
669 
670 		kfree(indir_desc);
671 		vq->desc_state[head].indir_desc = NULL;
672 	} else if (ctx) {
673 		*ctx = vq->desc_state[head].indir_desc;
674 	}
675 }
676 
more_used(const struct vring_virtqueue * vq)677 static inline bool more_used(const struct vring_virtqueue *vq)
678 {
679 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, vq->vring.used->idx);
680 }
681 
682 /**
683  * virtqueue_get_buf - get the next used buffer
684  * @vq: the struct virtqueue we're talking about.
685  * @len: the length written into the buffer
686  *
687  * If the device wrote data into the buffer, @len will be set to the
688  * amount written.  This means you don't need to clear the buffer
689  * beforehand to ensure there's no data leakage in the case of short
690  * writes.
691  *
692  * Caller must ensure we don't call this with other virtqueue
693  * operations at the same time (except where noted).
694  *
695  * Returns NULL if there are no used buffers, or the "data" token
696  * handed to virtqueue_add_*().
697  */
virtqueue_get_buf_ctx(struct virtqueue * _vq,unsigned int * len,void ** ctx)698 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
699 			    void **ctx)
700 {
701 	struct vring_virtqueue *vq = to_vvq(_vq);
702 	void *ret;
703 	unsigned int i;
704 	u16 last_used;
705 
706 	START_USE(vq);
707 
708 	if (unlikely(vq->broken)) {
709 		END_USE(vq);
710 		return NULL;
711 	}
712 
713 	if (!more_used(vq)) {
714 		pr_debug("No more buffers in queue\n");
715 		END_USE(vq);
716 		return NULL;
717 	}
718 
719 	/* Only get used array entries after they have been exposed by host. */
720 	virtio_rmb(vq->weak_barriers);
721 
722 	last_used = (vq->last_used_idx & (vq->vring.num - 1));
723 	i = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].id);
724 	*len = virtio32_to_cpu(_vq->vdev, vq->vring.used->ring[last_used].len);
725 
726 	if (unlikely(i >= vq->vring.num)) {
727 		BAD_RING(vq, "id %u out of range\n", i);
728 		return NULL;
729 	}
730 	if (unlikely(!vq->desc_state[i].data)) {
731 		BAD_RING(vq, "id %u is not a head!\n", i);
732 		return NULL;
733 	}
734 
735 	/* detach_buf clears data, so grab it now. */
736 	ret = vq->desc_state[i].data;
737 	detach_buf(vq, i, ctx);
738 	vq->last_used_idx++;
739 	/* If we expect an interrupt for the next entry, tell host
740 	 * by writing event index and flush out the write before
741 	 * the read in the next get_buf call. */
742 	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
743 		virtio_store_mb(vq->weak_barriers,
744 				&vring_used_event(&vq->vring),
745 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
746 
747 #ifdef DEBUG
748 	vq->last_add_time_valid = false;
749 #endif
750 
751 	END_USE(vq);
752 	return ret;
753 }
754 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
755 
virtqueue_get_buf(struct virtqueue * _vq,unsigned int * len)756 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
757 {
758 	return virtqueue_get_buf_ctx(_vq, len, NULL);
759 }
760 EXPORT_SYMBOL_GPL(virtqueue_get_buf);
761 /**
762  * virtqueue_disable_cb - disable callbacks
763  * @vq: the struct virtqueue we're talking about.
764  *
765  * Note that this is not necessarily synchronous, hence unreliable and only
766  * useful as an optimization.
767  *
768  * Unlike other operations, this need not be serialized.
769  */
virtqueue_disable_cb(struct virtqueue * _vq)770 void virtqueue_disable_cb(struct virtqueue *_vq)
771 {
772 	struct vring_virtqueue *vq = to_vvq(_vq);
773 
774 	if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
775 		vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
776 		if (!vq->event)
777 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
778 	}
779 
780 }
781 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
782 
783 /**
784  * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
785  * @vq: the struct virtqueue we're talking about.
786  *
787  * This re-enables callbacks; it returns current queue state
788  * in an opaque unsigned value. This value should be later tested by
789  * virtqueue_poll, to detect a possible race between the driver checking for
790  * more work, and enabling callbacks.
791  *
792  * Caller must ensure we don't call this with other virtqueue
793  * operations at the same time (except where noted).
794  */
virtqueue_enable_cb_prepare(struct virtqueue * _vq)795 unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
796 {
797 	struct vring_virtqueue *vq = to_vvq(_vq);
798 	u16 last_used_idx;
799 
800 	START_USE(vq);
801 
802 	/* We optimistically turn back on interrupts, then check if there was
803 	 * more to do. */
804 	/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
805 	 * either clear the flags bit or point the event index at the next
806 	 * entry. Always do both to keep code simple. */
807 	if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
808 		vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
809 		if (!vq->event)
810 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
811 	}
812 	vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, last_used_idx = vq->last_used_idx);
813 	END_USE(vq);
814 	return last_used_idx;
815 }
816 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
817 
818 /**
819  * virtqueue_poll - query pending used buffers
820  * @vq: the struct virtqueue we're talking about.
821  * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
822  *
823  * Returns "true" if there are pending used buffers in the queue.
824  *
825  * This does not need to be serialized.
826  */
virtqueue_poll(struct virtqueue * _vq,unsigned last_used_idx)827 bool virtqueue_poll(struct virtqueue *_vq, unsigned last_used_idx)
828 {
829 	struct vring_virtqueue *vq = to_vvq(_vq);
830 
831 	if (unlikely(vq->broken))
832 		return false;
833 
834 	virtio_mb(vq->weak_barriers);
835 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, vq->vring.used->idx);
836 }
837 EXPORT_SYMBOL_GPL(virtqueue_poll);
838 
839 /**
840  * virtqueue_enable_cb - restart callbacks after disable_cb.
841  * @vq: the struct virtqueue we're talking about.
842  *
843  * This re-enables callbacks; it returns "false" if there are pending
844  * buffers in the queue, to detect a possible race between the driver
845  * checking for more work, and enabling callbacks.
846  *
847  * Caller must ensure we don't call this with other virtqueue
848  * operations at the same time (except where noted).
849  */
virtqueue_enable_cb(struct virtqueue * _vq)850 bool virtqueue_enable_cb(struct virtqueue *_vq)
851 {
852 	unsigned last_used_idx = virtqueue_enable_cb_prepare(_vq);
853 	return !virtqueue_poll(_vq, last_used_idx);
854 }
855 EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
856 
857 /**
858  * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
859  * @vq: the struct virtqueue we're talking about.
860  *
861  * This re-enables callbacks but hints to the other side to delay
862  * interrupts until most of the available buffers have been processed;
863  * it returns "false" if there are many pending buffers in the queue,
864  * to detect a possible race between the driver checking for more work,
865  * and enabling callbacks.
866  *
867  * Caller must ensure we don't call this with other virtqueue
868  * operations at the same time (except where noted).
869  */
virtqueue_enable_cb_delayed(struct virtqueue * _vq)870 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
871 {
872 	struct vring_virtqueue *vq = to_vvq(_vq);
873 	u16 bufs;
874 
875 	START_USE(vq);
876 
877 	/* We optimistically turn back on interrupts, then check if there was
878 	 * more to do. */
879 	/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
880 	 * either clear the flags bit or point the event index at the next
881 	 * entry. Always update the event index to keep code simple. */
882 	if (vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
883 		vq->avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
884 		if (!vq->event)
885 			vq->vring.avail->flags = cpu_to_virtio16(_vq->vdev, vq->avail_flags_shadow);
886 	}
887 	/* TODO: tune this threshold */
888 	bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4;
889 
890 	virtio_store_mb(vq->weak_barriers,
891 			&vring_used_event(&vq->vring),
892 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
893 
894 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) {
895 		END_USE(vq);
896 		return false;
897 	}
898 
899 	END_USE(vq);
900 	return true;
901 }
902 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
903 
904 /**
905  * virtqueue_detach_unused_buf - detach first unused buffer
906  * @vq: the struct virtqueue we're talking about.
907  *
908  * Returns NULL or the "data" token handed to virtqueue_add_*().
909  * This is not valid on an active queue; it is useful only for device
910  * shutdown.
911  */
virtqueue_detach_unused_buf(struct virtqueue * _vq)912 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
913 {
914 	struct vring_virtqueue *vq = to_vvq(_vq);
915 	unsigned int i;
916 	void *buf;
917 
918 	START_USE(vq);
919 
920 	for (i = 0; i < vq->vring.num; i++) {
921 		if (!vq->desc_state[i].data)
922 			continue;
923 		/* detach_buf clears data, so grab it now. */
924 		buf = vq->desc_state[i].data;
925 		detach_buf(vq, i, NULL);
926 		vq->avail_idx_shadow--;
927 		vq->vring.avail->idx = cpu_to_virtio16(_vq->vdev, vq->avail_idx_shadow);
928 		END_USE(vq);
929 		return buf;
930 	}
931 	/* That should have freed everything. */
932 	BUG_ON(vq->vq.num_free != vq->vring.num);
933 
934 	END_USE(vq);
935 	return NULL;
936 }
937 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
938 
vring_interrupt(int irq,void * _vq)939 irqreturn_t vring_interrupt(int irq, void *_vq)
940 {
941 	struct vring_virtqueue *vq = to_vvq(_vq);
942 
943 	if (!more_used(vq)) {
944 		pr_debug("virtqueue interrupt with no work for %p\n", vq);
945 		return IRQ_NONE;
946 	}
947 
948 	if (unlikely(vq->broken))
949 		return IRQ_HANDLED;
950 
951 	pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
952 	if (vq->vq.callback)
953 		vq->vq.callback(&vq->vq);
954 
955 	return IRQ_HANDLED;
956 }
957 EXPORT_SYMBOL_GPL(vring_interrupt);
958 
__vring_new_virtqueue(unsigned int index,struct vring vring,struct virtio_device * vdev,bool weak_barriers,bool context,bool (* notify)(struct virtqueue *),void (* callback)(struct virtqueue *),const char * name)959 struct virtqueue *__vring_new_virtqueue(unsigned int index,
960 					struct vring vring,
961 					struct virtio_device *vdev,
962 					bool weak_barriers,
963 					bool context,
964 					bool (*notify)(struct virtqueue *),
965 					void (*callback)(struct virtqueue *),
966 					const char *name)
967 {
968 	unsigned int i;
969 	struct vring_virtqueue *vq;
970 
971 	vq = kmalloc(sizeof(*vq) + vring.num * sizeof(struct vring_desc_state),
972 		     GFP_KERNEL);
973 	if (!vq)
974 		return NULL;
975 
976 	vq->vring = vring;
977 	vq->vq.callback = callback;
978 	vq->vq.vdev = vdev;
979 	vq->vq.name = name;
980 	vq->vq.num_free = vring.num;
981 	vq->vq.index = index;
982 	vq->we_own_ring = false;
983 	vq->queue_dma_addr = 0;
984 	vq->queue_size_in_bytes = 0;
985 	vq->notify = notify;
986 	vq->weak_barriers = weak_barriers;
987 	vq->broken = false;
988 	vq->last_used_idx = 0;
989 	vq->avail_flags_shadow = 0;
990 	vq->avail_idx_shadow = 0;
991 	vq->num_added = 0;
992 	list_add_tail(&vq->vq.list, &vdev->vqs);
993 #ifdef DEBUG
994 	vq->in_use = false;
995 	vq->last_add_time_valid = false;
996 #endif
997 
998 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
999 		!context;
1000 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1001 
1002 	/* No callback?  Tell other side not to bother us. */
1003 	if (!callback) {
1004 		vq->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
1005 		if (!vq->event)
1006 			vq->vring.avail->flags = cpu_to_virtio16(vdev, vq->avail_flags_shadow);
1007 	}
1008 
1009 	/* Put everything in free lists. */
1010 	vq->free_head = 0;
1011 	for (i = 0; i < vring.num-1; i++)
1012 		vq->vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
1013 	memset(vq->desc_state, 0, vring.num * sizeof(struct vring_desc_state));
1014 
1015 	return &vq->vq;
1016 }
1017 EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
1018 
vring_alloc_queue(struct virtio_device * vdev,size_t size,dma_addr_t * dma_handle,gfp_t flag)1019 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
1020 			      dma_addr_t *dma_handle, gfp_t flag)
1021 {
1022 	if (vring_use_dma_api(vdev)) {
1023 		return dma_alloc_coherent(vdev->dev.parent, size,
1024 					  dma_handle, flag);
1025 	} else {
1026 		void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
1027 		if (queue) {
1028 			phys_addr_t phys_addr = virt_to_phys(queue);
1029 			*dma_handle = (dma_addr_t)phys_addr;
1030 
1031 			/*
1032 			 * Sanity check: make sure we dind't truncate
1033 			 * the address.  The only arches I can find that
1034 			 * have 64-bit phys_addr_t but 32-bit dma_addr_t
1035 			 * are certain non-highmem MIPS and x86
1036 			 * configurations, but these configurations
1037 			 * should never allocate physical pages above 32
1038 			 * bits, so this is fine.  Just in case, throw a
1039 			 * warning and abort if we end up with an
1040 			 * unrepresentable address.
1041 			 */
1042 			if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
1043 				free_pages_exact(queue, PAGE_ALIGN(size));
1044 				return NULL;
1045 			}
1046 		}
1047 		return queue;
1048 	}
1049 }
1050 
vring_free_queue(struct virtio_device * vdev,size_t size,void * queue,dma_addr_t dma_handle)1051 static void vring_free_queue(struct virtio_device *vdev, size_t size,
1052 			     void *queue, dma_addr_t dma_handle)
1053 {
1054 	if (vring_use_dma_api(vdev)) {
1055 		dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
1056 	} else {
1057 		free_pages_exact(queue, PAGE_ALIGN(size));
1058 	}
1059 }
1060 
vring_create_virtqueue(unsigned int index,unsigned int num,unsigned int vring_align,struct virtio_device * vdev,bool weak_barriers,bool may_reduce_num,bool context,bool (* notify)(struct virtqueue *),void (* callback)(struct virtqueue *),const char * name)1061 struct virtqueue *vring_create_virtqueue(
1062 	unsigned int index,
1063 	unsigned int num,
1064 	unsigned int vring_align,
1065 	struct virtio_device *vdev,
1066 	bool weak_barriers,
1067 	bool may_reduce_num,
1068 	bool context,
1069 	bool (*notify)(struct virtqueue *),
1070 	void (*callback)(struct virtqueue *),
1071 	const char *name)
1072 {
1073 	struct virtqueue *vq;
1074 	void *queue = NULL;
1075 	dma_addr_t dma_addr;
1076 	size_t queue_size_in_bytes;
1077 	struct vring vring;
1078 
1079 	/* We assume num is a power of 2. */
1080 	if (num & (num - 1)) {
1081 		dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
1082 		return NULL;
1083 	}
1084 
1085 	/* TODO: allocate each queue chunk individually */
1086 	for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
1087 		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1088 					  &dma_addr,
1089 					  GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1090 		if (queue)
1091 			break;
1092 		if (!may_reduce_num)
1093 			return NULL;
1094 	}
1095 
1096 	if (!num)
1097 		return NULL;
1098 
1099 	if (!queue) {
1100 		/* Try to get a single page. You are my only hope! */
1101 		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
1102 					  &dma_addr, GFP_KERNEL|__GFP_ZERO);
1103 	}
1104 	if (!queue)
1105 		return NULL;
1106 
1107 	queue_size_in_bytes = vring_size(num, vring_align);
1108 	vring_init(&vring, num, queue, vring_align);
1109 
1110 	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
1111 				   notify, callback, name);
1112 	if (!vq) {
1113 		vring_free_queue(vdev, queue_size_in_bytes, queue,
1114 				 dma_addr);
1115 		return NULL;
1116 	}
1117 
1118 	to_vvq(vq)->queue_dma_addr = dma_addr;
1119 	to_vvq(vq)->queue_size_in_bytes = queue_size_in_bytes;
1120 	to_vvq(vq)->we_own_ring = true;
1121 
1122 	return vq;
1123 }
1124 EXPORT_SYMBOL_GPL(vring_create_virtqueue);
1125 
vring_new_virtqueue(unsigned int index,unsigned int num,unsigned int vring_align,struct virtio_device * vdev,bool weak_barriers,bool context,void * pages,bool (* notify)(struct virtqueue * vq),void (* callback)(struct virtqueue * vq),const char * name)1126 struct virtqueue *vring_new_virtqueue(unsigned int index,
1127 				      unsigned int num,
1128 				      unsigned int vring_align,
1129 				      struct virtio_device *vdev,
1130 				      bool weak_barriers,
1131 				      bool context,
1132 				      void *pages,
1133 				      bool (*notify)(struct virtqueue *vq),
1134 				      void (*callback)(struct virtqueue *vq),
1135 				      const char *name)
1136 {
1137 	struct vring vring;
1138 	vring_init(&vring, num, pages, vring_align);
1139 	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
1140 				     notify, callback, name);
1141 }
1142 EXPORT_SYMBOL_GPL(vring_new_virtqueue);
1143 
vring_del_virtqueue(struct virtqueue * _vq)1144 void vring_del_virtqueue(struct virtqueue *_vq)
1145 {
1146 	struct vring_virtqueue *vq = to_vvq(_vq);
1147 
1148 	if (vq->we_own_ring) {
1149 		vring_free_queue(vq->vq.vdev, vq->queue_size_in_bytes,
1150 				 vq->vring.desc, vq->queue_dma_addr);
1151 	}
1152 	list_del(&_vq->list);
1153 	kfree(vq);
1154 }
1155 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
1156 
1157 /* Manipulates transport-specific feature bits. */
vring_transport_features(struct virtio_device * vdev)1158 void vring_transport_features(struct virtio_device *vdev)
1159 {
1160 	unsigned int i;
1161 
1162 	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
1163 		switch (i) {
1164 		case VIRTIO_RING_F_INDIRECT_DESC:
1165 			break;
1166 		case VIRTIO_RING_F_EVENT_IDX:
1167 			break;
1168 		case VIRTIO_F_VERSION_1:
1169 			break;
1170 		case VIRTIO_F_IOMMU_PLATFORM:
1171 			break;
1172 		default:
1173 			/* We don't understand this bit. */
1174 			__virtio_clear_bit(vdev, i);
1175 		}
1176 	}
1177 }
1178 EXPORT_SYMBOL_GPL(vring_transport_features);
1179 
1180 /**
1181  * virtqueue_get_vring_size - return the size of the virtqueue's vring
1182  * @vq: the struct virtqueue containing the vring of interest.
1183  *
1184  * Returns the size of the vring.  This is mainly used for boasting to
1185  * userspace.  Unlike other operations, this need not be serialized.
1186  */
virtqueue_get_vring_size(struct virtqueue * _vq)1187 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
1188 {
1189 
1190 	struct vring_virtqueue *vq = to_vvq(_vq);
1191 
1192 	return vq->vring.num;
1193 }
1194 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
1195 
virtqueue_is_broken(struct virtqueue * _vq)1196 bool virtqueue_is_broken(struct virtqueue *_vq)
1197 {
1198 	struct vring_virtqueue *vq = to_vvq(_vq);
1199 
1200 	return READ_ONCE(vq->broken);
1201 }
1202 EXPORT_SYMBOL_GPL(virtqueue_is_broken);
1203 
1204 /*
1205  * This should prevent the device from being used, allowing drivers to
1206  * recover.  You may need to grab appropriate locks to flush.
1207  */
virtio_break_device(struct virtio_device * dev)1208 void virtio_break_device(struct virtio_device *dev)
1209 {
1210 	struct virtqueue *_vq;
1211 
1212 	list_for_each_entry(_vq, &dev->vqs, list) {
1213 		struct vring_virtqueue *vq = to_vvq(_vq);
1214 
1215 		/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
1216 		WRITE_ONCE(vq->broken, true);
1217 	}
1218 }
1219 EXPORT_SYMBOL_GPL(virtio_break_device);
1220 
virtqueue_get_desc_addr(struct virtqueue * _vq)1221 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
1222 {
1223 	struct vring_virtqueue *vq = to_vvq(_vq);
1224 
1225 	BUG_ON(!vq->we_own_ring);
1226 
1227 	return vq->queue_dma_addr;
1228 }
1229 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
1230 
virtqueue_get_avail_addr(struct virtqueue * _vq)1231 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
1232 {
1233 	struct vring_virtqueue *vq = to_vvq(_vq);
1234 
1235 	BUG_ON(!vq->we_own_ring);
1236 
1237 	return vq->queue_dma_addr +
1238 		((char *)vq->vring.avail - (char *)vq->vring.desc);
1239 }
1240 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
1241 
virtqueue_get_used_addr(struct virtqueue * _vq)1242 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
1243 {
1244 	struct vring_virtqueue *vq = to_vvq(_vq);
1245 
1246 	BUG_ON(!vq->we_own_ring);
1247 
1248 	return vq->queue_dma_addr +
1249 		((char *)vq->vring.used - (char *)vq->vring.desc);
1250 }
1251 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
1252 
virtqueue_get_vring(struct virtqueue * vq)1253 const struct vring *virtqueue_get_vring(struct virtqueue *vq)
1254 {
1255 	return &to_vvq(vq)->vring;
1256 }
1257 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
1258 
1259 MODULE_LICENSE("GPL");
1260