324 static u64 fuse_get_unique(struct fuse_iqueue *fiq)  in fuse_get_unique()  argument
326 	return ++fiq->reqctr;  in fuse_get_unique()
329 static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)  in queue_request()  argument
333 	list_add_tail(&req->list, &fiq->pending);  in queue_request()
334 	wake_up(&fiq->waitq);  in queue_request()
335 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in queue_request()
341 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_queue_forget()  local
346 	spin_lock(&fiq->lock);  in fuse_queue_forget()
347 	if (fiq->connected) {  in fuse_queue_forget()
348 		fiq->forget_list_tail->next = forget;  in fuse_queue_forget()
349 		fiq->forget_list_tail = forget;  in fuse_queue_forget()
350 		wake_up(&fiq->waitq);  in fuse_queue_forget()
351 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_queue_forget()
355 	spin_unlock(&fiq->lock);  in fuse_queue_forget()
363 		struct fuse_iqueue *fiq = &fc->iq;  in flush_bg_queue()  local
368 		spin_lock(&fiq->lock);  in flush_bg_queue()
369 		req->in.h.unique = fuse_get_unique(fiq);  in flush_bg_queue()
370 		queue_request(fiq, req);  in flush_bg_queue()
371 		spin_unlock(&fiq->lock);  in flush_bg_queue()
385 	struct fuse_iqueue *fiq = &fc->iq;  in request_end()  local
390 	spin_lock(&fiq->lock);  in request_end()
392 	spin_unlock(&fiq->lock);  in request_end()
428 static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)  in queue_interrupt()  argument
430 	spin_lock(&fiq->lock);  in queue_interrupt()
432 		spin_unlock(&fiq->lock);  in queue_interrupt()
436 		list_add_tail(&req->intr_entry, &fiq->interrupts);  in queue_interrupt()
437 		wake_up(&fiq->waitq);  in queue_interrupt()
439 	spin_unlock(&fiq->lock);  in queue_interrupt()
440 	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in queue_interrupt()
445 	struct fuse_iqueue *fiq = &fc->iq;  in request_wait_answer()  local
459 			queue_interrupt(fiq, req);  in request_wait_answer()
469 		spin_lock(&fiq->lock);  in request_wait_answer()
473 			spin_unlock(&fiq->lock);  in request_wait_answer()
478 		spin_unlock(&fiq->lock);  in request_wait_answer()
490 	struct fuse_iqueue *fiq = &fc->iq;  in __fuse_request_send()  local
493 	spin_lock(&fiq->lock);  in __fuse_request_send()
494 	if (!fiq->connected) {  in __fuse_request_send()
495 		spin_unlock(&fiq->lock);  in __fuse_request_send()
498 		req->in.h.unique = fuse_get_unique(fiq);  in __fuse_request_send()
499 		queue_request(fiq, req);  in __fuse_request_send()
503 		spin_unlock(&fiq->lock);  in __fuse_request_send()
632 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_request_send_notify_reply()  local
636 	spin_lock(&fiq->lock);  in fuse_request_send_notify_reply()
637 	if (fiq->connected) {  in fuse_request_send_notify_reply()
638 		queue_request(fiq, req);  in fuse_request_send_notify_reply()
641 	spin_unlock(&fiq->lock);  in fuse_request_send_notify_reply()
1092 static int forget_pending(struct fuse_iqueue *fiq)  in forget_pending()  argument
1094 	return fiq->forget_list_head.next != NULL;  in forget_pending()
1097 static int request_pending(struct fuse_iqueue *fiq)  in request_pending()  argument
1099 	return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||  in request_pending()
1100 		forget_pending(fiq);  in request_pending()
1111 static int fuse_read_interrupt(struct fuse_iqueue *fiq,  in fuse_read_interrupt()  argument
1114 __releases(fiq->lock)  in fuse_read_interrupt()
1122 	req->intr_unique = fuse_get_unique(fiq);  in fuse_read_interrupt()
1130 	spin_unlock(&fiq->lock);  in fuse_read_interrupt()
1142 static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,  in dequeue_forget()  argument
1146 	struct fuse_forget_link *head = fiq->forget_list_head.next;  in dequeue_forget()
1153 	fiq->forget_list_head.next = *newhead;  in dequeue_forget()
1155 	if (fiq->forget_list_head.next == NULL)  in dequeue_forget()
1156 		fiq->forget_list_tail = &fiq->forget_list_head;  in dequeue_forget()
1164 static int fuse_read_single_forget(struct fuse_iqueue *fiq,  in fuse_read_single_forget()  argument
1167 __releases(fiq->lock)  in fuse_read_single_forget()
1170 	struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);  in fuse_read_single_forget()
1177 		.unique = fuse_get_unique(fiq),  in fuse_read_single_forget()
1181 	spin_unlock(&fiq->lock);  in fuse_read_single_forget()
1197 static int fuse_read_batch_forget(struct fuse_iqueue *fiq,  in fuse_read_batch_forget()  argument
1199 __releases(fiq->lock)  in fuse_read_batch_forget()
1208 		.unique = fuse_get_unique(fiq),  in fuse_read_batch_forget()
1213 		spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1218 	head = dequeue_forget(fiq, max_forgets, &count);  in fuse_read_batch_forget()
1219 	spin_unlock(&fiq->lock);  in fuse_read_batch_forget()
1246 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,  in fuse_read_forget()  argument
1249 __releases(fiq->lock)  in fuse_read_forget()
1251 	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)  in fuse_read_forget()
1252 		return fuse_read_single_forget(fiq, cs, nbytes);  in fuse_read_forget()
1254 		return fuse_read_batch_forget(fiq, cs, nbytes);  in fuse_read_forget()
1271 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_dev_do_read()  local
1279 		spin_lock(&fiq->lock);  in fuse_dev_do_read()
1280 		if (!fiq->connected || request_pending(fiq))  in fuse_dev_do_read()
1282 		spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1286 		err = wait_event_interruptible_exclusive(fiq->waitq,  in fuse_dev_do_read()
1287 				!fiq->connected || request_pending(fiq));  in fuse_dev_do_read()
1292 	if (!fiq->connected) {  in fuse_dev_do_read()
1297 	if (!list_empty(&fiq->interrupts)) {  in fuse_dev_do_read()
1298 		req = list_entry(fiq->interrupts.next, struct fuse_req,  in fuse_dev_do_read()
1300 		return fuse_read_interrupt(fiq, cs, nbytes, req);  in fuse_dev_do_read()
1303 	if (forget_pending(fiq)) {  in fuse_dev_do_read()
1304 		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)  in fuse_dev_do_read()
1305 			return fuse_read_forget(fc, fiq, cs, nbytes);  in fuse_dev_do_read()
1307 		if (fiq->forget_batch <= -8)  in fuse_dev_do_read()
1308 			fiq->forget_batch = 16;  in fuse_dev_do_read()
1311 	req = list_entry(fiq->pending.next, struct fuse_req, list);  in fuse_dev_do_read()
1314 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
1367 		queue_interrupt(fiq, req);  in fuse_dev_do_read()
1380 	spin_unlock(&fiq->lock);  in fuse_dev_do_read()
2088 	struct fuse_iqueue *fiq;  in fuse_dev_poll()  local
2094 	fiq = &fud->fc->iq;  in fuse_dev_poll()
2095 	poll_wait(file, &fiq->waitq, wait);  in fuse_dev_poll()
2097 	spin_lock(&fiq->lock);  in fuse_dev_poll()
2098 	if (!fiq->connected)  in fuse_dev_poll()
2100 	else if (request_pending(fiq))  in fuse_dev_poll()
2102 	spin_unlock(&fiq->lock);  in fuse_dev_poll()
2159 	struct fuse_iqueue *fiq = &fc->iq;  in fuse_abort_conn()  local
2193 		spin_lock(&fiq->lock);  in fuse_abort_conn()
2194 		fiq->connected = 0;  in fuse_abort_conn()
2195 		list_for_each_entry(req, &fiq->pending, list)  in fuse_abort_conn()
2197 		list_splice_tail_init(&fiq->pending, &to_end);  in fuse_abort_conn()
2198 		while (forget_pending(fiq))  in fuse_abort_conn()
2199 			kfree(dequeue_forget(fiq, 1, NULL));  in fuse_abort_conn()
2200 		wake_up_all(&fiq->waitq);  in fuse_abort_conn()
2201 		spin_unlock(&fiq->lock);  in fuse_abort_conn()
2202 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);  in fuse_abort_conn()