1 /*
2  * linux/fs/nfs/write.c
3  *
4  * Write file data over NFS.
5  *
6  * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
7  */
8 
9 #include <linux/types.h>
10 #include <linux/slab.h>
11 #include <linux/mm.h>
12 #include <linux/pagemap.h>
13 #include <linux/file.h>
14 #include <linux/writeback.h>
15 #include <linux/swap.h>
16 #include <linux/migrate.h>
17 
18 #include <linux/sunrpc/clnt.h>
19 #include <linux/nfs_fs.h>
20 #include <linux/nfs_mount.h>
21 #include <linux/nfs_page.h>
22 #include <linux/backing-dev.h>
23 #include <linux/export.h>
24 #include <linux/freezer.h>
25 #include <linux/wait.h>
26 #include <linux/iversion.h>
27 
28 #include <linux/uaccess.h>
29 
30 #include "delegation.h"
31 #include "internal.h"
32 #include "iostat.h"
33 #include "nfs4_fs.h"
34 #include "fscache.h"
35 #include "pnfs.h"
36 
37 #include "nfstrace.h"
38 
39 #define NFSDBG_FACILITY		NFSDBG_PAGECACHE
40 
41 #define MIN_POOL_WRITE		(32)
42 #define MIN_POOL_COMMIT		(4)
43 
44 struct nfs_io_completion {
45 	void (*complete)(void *data);
46 	void *data;
47 	struct kref refcount;
48 };
49 
50 /*
51  * Local function declarations
52  */
53 static void nfs_redirty_request(struct nfs_page *req);
54 static const struct rpc_call_ops nfs_commit_ops;
55 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
56 static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
57 static const struct nfs_rw_ops nfs_rw_write_ops;
58 static void nfs_clear_request_commit(struct nfs_page *req);
59 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
60 				      struct inode *inode);
61 static struct nfs_page *
62 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
63 						struct page *page);
64 
65 static struct kmem_cache *nfs_wdata_cachep;
66 static mempool_t *nfs_wdata_mempool;
67 static struct kmem_cache *nfs_cdata_cachep;
68 static mempool_t *nfs_commit_mempool;
69 
nfs_commitdata_alloc(bool never_fail)70 struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
71 {
72 	struct nfs_commit_data *p;
73 
74 	if (never_fail)
75 		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
76 	else {
77 		/* It is OK to do some reclaim, not no safe to wait
78 		 * for anything to be returned to the pool.
79 		 * mempool_alloc() cannot handle that particular combination,
80 		 * so we need two separate attempts.
81 		 */
82 		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
83 		if (!p)
84 			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
85 					     __GFP_NOWARN | __GFP_NORETRY);
86 		if (!p)
87 			return NULL;
88 	}
89 
90 	memset(p, 0, sizeof(*p));
91 	INIT_LIST_HEAD(&p->pages);
92 	return p;
93 }
94 EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
95 
nfs_commit_free(struct nfs_commit_data * p)96 void nfs_commit_free(struct nfs_commit_data *p)
97 {
98 	mempool_free(p, nfs_commit_mempool);
99 }
100 EXPORT_SYMBOL_GPL(nfs_commit_free);
101 
nfs_writehdr_alloc(void)102 static struct nfs_pgio_header *nfs_writehdr_alloc(void)
103 {
104 	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_NOIO);
105 
106 	memset(p, 0, sizeof(*p));
107 	p->rw_mode = FMODE_WRITE;
108 	return p;
109 }
110 
nfs_writehdr_free(struct nfs_pgio_header * hdr)111 static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
112 {
113 	mempool_free(hdr, nfs_wdata_mempool);
114 }
115 
nfs_io_completion_alloc(gfp_t gfp_flags)116 static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
117 {
118 	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
119 }
120 
nfs_io_completion_init(struct nfs_io_completion * ioc,void (* complete)(void *),void * data)121 static void nfs_io_completion_init(struct nfs_io_completion *ioc,
122 		void (*complete)(void *), void *data)
123 {
124 	ioc->complete = complete;
125 	ioc->data = data;
126 	kref_init(&ioc->refcount);
127 }
128 
nfs_io_completion_release(struct kref * kref)129 static void nfs_io_completion_release(struct kref *kref)
130 {
131 	struct nfs_io_completion *ioc = container_of(kref,
132 			struct nfs_io_completion, refcount);
133 	ioc->complete(ioc->data);
134 	kfree(ioc);
135 }
136 
nfs_io_completion_get(struct nfs_io_completion * ioc)137 static void nfs_io_completion_get(struct nfs_io_completion *ioc)
138 {
139 	if (ioc != NULL)
140 		kref_get(&ioc->refcount);
141 }
142 
nfs_io_completion_put(struct nfs_io_completion * ioc)143 static void nfs_io_completion_put(struct nfs_io_completion *ioc)
144 {
145 	if (ioc != NULL)
146 		kref_put(&ioc->refcount, nfs_io_completion_release);
147 }
148 
149 static struct nfs_page *
nfs_page_private_request(struct page * page)150 nfs_page_private_request(struct page *page)
151 {
152 	if (!PagePrivate(page))
153 		return NULL;
154 	return (struct nfs_page *)page_private(page);
155 }
156 
157 /*
158  * nfs_page_find_head_request_locked - find head request associated with @page
159  *
160  * must be called while holding the inode lock.
161  *
162  * returns matching head request with reference held, or NULL if not found.
163  */
164 static struct nfs_page *
nfs_page_find_private_request(struct page * page)165 nfs_page_find_private_request(struct page *page)
166 {
167 	struct address_space *mapping = page_file_mapping(page);
168 	struct nfs_page *req;
169 
170 	if (!PagePrivate(page))
171 		return NULL;
172 	spin_lock(&mapping->private_lock);
173 	req = nfs_page_private_request(page);
174 	if (req) {
175 		WARN_ON_ONCE(req->wb_head != req);
176 		kref_get(&req->wb_kref);
177 	}
178 	spin_unlock(&mapping->private_lock);
179 	return req;
180 }
181 
182 static struct nfs_page *
nfs_page_find_swap_request(struct page * page)183 nfs_page_find_swap_request(struct page *page)
184 {
185 	struct inode *inode = page_file_mapping(page)->host;
186 	struct nfs_inode *nfsi = NFS_I(inode);
187 	struct nfs_page *req = NULL;
188 	if (!PageSwapCache(page))
189 		return NULL;
190 	mutex_lock(&nfsi->commit_mutex);
191 	if (PageSwapCache(page)) {
192 		req = nfs_page_search_commits_for_head_request_locked(nfsi,
193 			page);
194 		if (req) {
195 			WARN_ON_ONCE(req->wb_head != req);
196 			kref_get(&req->wb_kref);
197 		}
198 	}
199 	mutex_unlock(&nfsi->commit_mutex);
200 	return req;
201 }
202 
203 /*
204  * nfs_page_find_head_request - find head request associated with @page
205  *
206  * returns matching head request with reference held, or NULL if not found.
207  */
nfs_page_find_head_request(struct page * page)208 static struct nfs_page *nfs_page_find_head_request(struct page *page)
209 {
210 	struct nfs_page *req;
211 
212 	req = nfs_page_find_private_request(page);
213 	if (!req)
214 		req = nfs_page_find_swap_request(page);
215 	return req;
216 }
217 
218 /* Adjust the file length if we're writing beyond the end */
nfs_grow_file(struct page * page,unsigned int offset,unsigned int count)219 static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
220 {
221 	struct inode *inode = page_file_mapping(page)->host;
222 	loff_t end, i_size;
223 	pgoff_t end_index;
224 
225 	spin_lock(&inode->i_lock);
226 	i_size = i_size_read(inode);
227 	end_index = (i_size - 1) >> PAGE_SHIFT;
228 	if (i_size > 0 && page_index(page) < end_index)
229 		goto out;
230 	end = page_file_offset(page) + ((loff_t)offset+count);
231 	if (i_size >= end)
232 		goto out;
233 	i_size_write(inode, end);
234 	NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
235 	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
236 out:
237 	spin_unlock(&inode->i_lock);
238 }
239 
240 /* A writeback failed: mark the page as bad, and invalidate the page cache */
nfs_set_pageerror(struct address_space * mapping)241 static void nfs_set_pageerror(struct address_space *mapping)
242 {
243 	struct inode *inode = mapping->host;
244 
245 	nfs_zap_mapping(mapping->host, mapping);
246 	/* Force file size revalidation */
247 	spin_lock(&inode->i_lock);
248 	NFS_I(inode)->cache_validity |= NFS_INO_REVAL_FORCED |
249 					NFS_INO_REVAL_PAGECACHE |
250 					NFS_INO_INVALID_SIZE;
251 	spin_unlock(&inode->i_lock);
252 }
253 
254 /*
255  * nfs_page_group_search_locked
256  * @head - head request of page group
257  * @page_offset - offset into page
258  *
259  * Search page group with head @head to find a request that contains the
260  * page offset @page_offset.
261  *
262  * Returns a pointer to the first matching nfs request, or NULL if no
263  * match is found.
264  *
265  * Must be called with the page group lock held
266  */
267 static struct nfs_page *
nfs_page_group_search_locked(struct nfs_page * head,unsigned int page_offset)268 nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
269 {
270 	struct nfs_page *req;
271 
272 	req = head;
273 	do {
274 		if (page_offset >= req->wb_pgbase &&
275 		    page_offset < (req->wb_pgbase + req->wb_bytes))
276 			return req;
277 
278 		req = req->wb_this_page;
279 	} while (req != head);
280 
281 	return NULL;
282 }
283 
284 /*
285  * nfs_page_group_covers_page
286  * @head - head request of page group
287  *
288  * Return true if the page group with head @head covers the whole page,
289  * returns false otherwise
290  */
nfs_page_group_covers_page(struct nfs_page * req)291 static bool nfs_page_group_covers_page(struct nfs_page *req)
292 {
293 	struct nfs_page *tmp;
294 	unsigned int pos = 0;
295 	unsigned int len = nfs_page_length(req->wb_page);
296 
297 	nfs_page_group_lock(req);
298 
299 	for (;;) {
300 		tmp = nfs_page_group_search_locked(req->wb_head, pos);
301 		if (!tmp)
302 			break;
303 		pos = tmp->wb_pgbase + tmp->wb_bytes;
304 	}
305 
306 	nfs_page_group_unlock(req);
307 	return pos >= len;
308 }
309 
310 /* We can set the PG_uptodate flag if we see that a write request
311  * covers the full page.
312  */
nfs_mark_uptodate(struct nfs_page * req)313 static void nfs_mark_uptodate(struct nfs_page *req)
314 {
315 	if (PageUptodate(req->wb_page))
316 		return;
317 	if (!nfs_page_group_covers_page(req))
318 		return;
319 	SetPageUptodate(req->wb_page);
320 }
321 
wb_priority(struct writeback_control * wbc)322 static int wb_priority(struct writeback_control *wbc)
323 {
324 	int ret = 0;
325 
326 	if (wbc->sync_mode == WB_SYNC_ALL)
327 		ret = FLUSH_COND_STABLE;
328 	return ret;
329 }
330 
331 /*
332  * NFS congestion control
333  */
334 
335 int nfs_congestion_kb;
336 
337 #define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
338 #define NFS_CONGESTION_OFF_THRESH	\
339 	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
340 
nfs_set_page_writeback(struct page * page)341 static void nfs_set_page_writeback(struct page *page)
342 {
343 	struct inode *inode = page_file_mapping(page)->host;
344 	struct nfs_server *nfss = NFS_SERVER(inode);
345 	int ret = test_set_page_writeback(page);
346 
347 	WARN_ON_ONCE(ret != 0);
348 
349 	if (atomic_long_inc_return(&nfss->writeback) >
350 			NFS_CONGESTION_ON_THRESH)
351 		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
352 }
353 
nfs_end_page_writeback(struct nfs_page * req)354 static void nfs_end_page_writeback(struct nfs_page *req)
355 {
356 	struct inode *inode = page_file_mapping(req->wb_page)->host;
357 	struct nfs_server *nfss = NFS_SERVER(inode);
358 	bool is_done;
359 
360 	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
361 	nfs_unlock_request(req);
362 	if (!is_done)
363 		return;
364 
365 	end_page_writeback(req->wb_page);
366 	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
367 		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
368 }
369 
370 /*
371  * nfs_unroll_locks_and_wait -  unlock all newly locked reqs and wait on @req
372  *
373  * this is a helper function for nfs_lock_and_join_requests
374  *
375  * @inode - inode associated with request page group, must be holding inode lock
376  * @head  - head request of page group, must be holding head lock
377  * @req   - request that couldn't lock and needs to wait on the req bit lock
378  *
379  * NOTE: this must be called holding page_group bit lock
380  *       which will be released before returning.
381  *
382  * returns 0 on success, < 0 on error.
383  */
384 static void
nfs_unroll_locks(struct inode * inode,struct nfs_page * head,struct nfs_page * req)385 nfs_unroll_locks(struct inode *inode, struct nfs_page *head,
386 			  struct nfs_page *req)
387 {
388 	struct nfs_page *tmp;
389 
390 	/* relinquish all the locks successfully grabbed this run */
391 	for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) {
392 		if (!kref_read(&tmp->wb_kref))
393 			continue;
394 		nfs_unlock_and_release_request(tmp);
395 	}
396 }
397 
398 /*
399  * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
400  *
401  * @destroy_list - request list (using wb_this_page) terminated by @old_head
402  * @old_head - the old head of the list
403  *
404  * All subrequests must be locked and removed from all lists, so at this point
405  * they are only "active" in this function, and possibly in nfs_wait_on_request
406  * with a reference held by some other context.
407  */
408 static void
nfs_destroy_unlinked_subrequests(struct nfs_page * destroy_list,struct nfs_page * old_head,struct inode * inode)409 nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
410 				 struct nfs_page *old_head,
411 				 struct inode *inode)
412 {
413 	while (destroy_list) {
414 		struct nfs_page *subreq = destroy_list;
415 
416 		destroy_list = (subreq->wb_this_page == old_head) ?
417 				   NULL : subreq->wb_this_page;
418 
419 		/* Note: lock subreq in order to change subreq->wb_head */
420 		nfs_page_set_headlock(subreq);
421 		WARN_ON_ONCE(old_head != subreq->wb_head);
422 
423 		/* make sure old group is not used */
424 		subreq->wb_this_page = subreq;
425 		subreq->wb_head = subreq;
426 
427 		clear_bit(PG_REMOVE, &subreq->wb_flags);
428 
429 		/* Note: races with nfs_page_group_destroy() */
430 		if (!kref_read(&subreq->wb_kref)) {
431 			/* Check if we raced with nfs_page_group_destroy() */
432 			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) {
433 				nfs_page_clear_headlock(subreq);
434 				nfs_free_request(subreq);
435 			} else
436 				nfs_page_clear_headlock(subreq);
437 			continue;
438 		}
439 		nfs_page_clear_headlock(subreq);
440 
441 		nfs_release_request(old_head);
442 
443 		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
444 			nfs_release_request(subreq);
445 			atomic_long_dec(&NFS_I(inode)->nrequests);
446 		}
447 
448 		/* subreq is now totally disconnected from page group or any
449 		 * write / commit lists. last chance to wake any waiters */
450 		nfs_unlock_and_release_request(subreq);
451 	}
452 }
453 
454 /*
455  * nfs_lock_and_join_requests - join all subreqs to the head req and return
456  *                              a locked reference, cancelling any pending
457  *                              operations for this page.
458  *
459  * @page - the page used to lookup the "page group" of nfs_page structures
460  *
461  * This function joins all sub requests to the head request by first
462  * locking all requests in the group, cancelling any pending operations
463  * and finally updating the head request to cover the whole range covered by
464  * the (former) group.  All subrequests are removed from any write or commit
465  * lists, unlinked from the group and destroyed.
466  *
467  * Returns a locked, referenced pointer to the head request - which after
468  * this call is guaranteed to be the only request associated with the page.
469  * Returns NULL if no requests are found for @page, or a ERR_PTR if an
470  * error was encountered.
471  */
472 static struct nfs_page *
nfs_lock_and_join_requests(struct page * page)473 nfs_lock_and_join_requests(struct page *page)
474 {
475 	struct inode *inode = page_file_mapping(page)->host;
476 	struct nfs_page *head, *subreq;
477 	struct nfs_page *destroy_list = NULL;
478 	unsigned int total_bytes;
479 	int ret;
480 
481 try_again:
482 	/*
483 	 * A reference is taken only on the head request which acts as a
484 	 * reference to the whole page group - the group will not be destroyed
485 	 * until the head reference is released.
486 	 */
487 	head = nfs_page_find_head_request(page);
488 	if (!head)
489 		return NULL;
490 
491 	/* lock the page head first in order to avoid an ABBA inefficiency */
492 	if (!nfs_lock_request(head)) {
493 		ret = nfs_wait_on_request(head);
494 		nfs_release_request(head);
495 		if (ret < 0)
496 			return ERR_PTR(ret);
497 		goto try_again;
498 	}
499 
500 	/* Ensure that nobody removed the request before we locked it */
501 	if (head != nfs_page_private_request(page) && !PageSwapCache(page)) {
502 		nfs_unlock_and_release_request(head);
503 		goto try_again;
504 	}
505 
506 	ret = nfs_page_group_lock(head);
507 	if (ret < 0)
508 		goto release_request;
509 
510 	/* lock each request in the page group */
511 	total_bytes = head->wb_bytes;
512 	for (subreq = head->wb_this_page; subreq != head;
513 			subreq = subreq->wb_this_page) {
514 
515 		if (!kref_get_unless_zero(&subreq->wb_kref)) {
516 			if (subreq->wb_offset == head->wb_offset + total_bytes)
517 				total_bytes += subreq->wb_bytes;
518 			continue;
519 		}
520 
521 		while (!nfs_lock_request(subreq)) {
522 			/*
523 			 * Unlock page to allow nfs_page_group_sync_on_bit()
524 			 * to succeed
525 			 */
526 			nfs_page_group_unlock(head);
527 			ret = nfs_wait_on_request(subreq);
528 			if (!ret)
529 				ret = nfs_page_group_lock(head);
530 			if (ret < 0) {
531 				nfs_unroll_locks(inode, head, subreq);
532 				nfs_release_request(subreq);
533 				goto release_request;
534 			}
535 		}
536 		/*
537 		 * Subrequests are always contiguous, non overlapping
538 		 * and in order - but may be repeated (mirrored writes).
539 		 */
540 		if (subreq->wb_offset == (head->wb_offset + total_bytes)) {
541 			/* keep track of how many bytes this group covers */
542 			total_bytes += subreq->wb_bytes;
543 		} else if (WARN_ON_ONCE(subreq->wb_offset < head->wb_offset ||
544 			    ((subreq->wb_offset + subreq->wb_bytes) >
545 			     (head->wb_offset + total_bytes)))) {
546 			nfs_page_group_unlock(head);
547 			nfs_unroll_locks(inode, head, subreq);
548 			nfs_unlock_and_release_request(subreq);
549 			ret = -EIO;
550 			goto release_request;
551 		}
552 	}
553 
554 	/* Now that all requests are locked, make sure they aren't on any list.
555 	 * Commit list removal accounting is done after locks are dropped */
556 	subreq = head;
557 	do {
558 		nfs_clear_request_commit(subreq);
559 		subreq = subreq->wb_this_page;
560 	} while (subreq != head);
561 
562 	/* unlink subrequests from head, destroy them later */
563 	if (head->wb_this_page != head) {
564 		/* destroy list will be terminated by head */
565 		destroy_list = head->wb_this_page;
566 		head->wb_this_page = head;
567 
568 		/* change head request to cover whole range that
569 		 * the former page group covered */
570 		head->wb_bytes = total_bytes;
571 	}
572 
573 	/* Postpone destruction of this request */
574 	if (test_and_clear_bit(PG_REMOVE, &head->wb_flags)) {
575 		set_bit(PG_INODE_REF, &head->wb_flags);
576 		kref_get(&head->wb_kref);
577 		atomic_long_inc(&NFS_I(inode)->nrequests);
578 	}
579 
580 	nfs_page_group_unlock(head);
581 
582 	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
583 
584 	/* Did we lose a race with nfs_inode_remove_request()? */
585 	if (!(PagePrivate(page) || PageSwapCache(page))) {
586 		nfs_unlock_and_release_request(head);
587 		return NULL;
588 	}
589 
590 	/* still holds ref on head from nfs_page_find_head_request
591 	 * and still has lock on head from lock loop */
592 	return head;
593 
594 release_request:
595 	nfs_unlock_and_release_request(head);
596 	return ERR_PTR(ret);
597 }
598 
nfs_write_error_remove_page(struct nfs_page * req)599 static void nfs_write_error_remove_page(struct nfs_page *req)
600 {
601 	nfs_end_page_writeback(req);
602 	generic_error_remove_page(page_file_mapping(req->wb_page),
603 				  req->wb_page);
604 	nfs_release_request(req);
605 }
606 
607 static bool
nfs_error_is_fatal_on_server(int err)608 nfs_error_is_fatal_on_server(int err)
609 {
610 	switch (err) {
611 	case 0:
612 	case -ERESTARTSYS:
613 	case -EINTR:
614 		return false;
615 	}
616 	return nfs_error_is_fatal(err);
617 }
618 
619 /*
620  * Find an associated nfs write request, and prepare to flush it out
621  * May return an error if the user signalled nfs_wait_on_request().
622  */
nfs_page_async_flush(struct nfs_pageio_descriptor * pgio,struct page * page)623 static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
624 				struct page *page)
625 {
626 	struct nfs_page *req;
627 	int ret = 0;
628 
629 	req = nfs_lock_and_join_requests(page);
630 	if (!req)
631 		goto out;
632 	ret = PTR_ERR(req);
633 	if (IS_ERR(req))
634 		goto out;
635 
636 	nfs_set_page_writeback(page);
637 	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
638 
639 	ret = req->wb_context->error;
640 	/* If there is a fatal error that covers this write, just exit */
641 	if (nfs_error_is_fatal_on_server(ret))
642 		goto out_launder;
643 
644 	ret = 0;
645 	if (!nfs_pageio_add_request(pgio, req)) {
646 		ret = pgio->pg_error;
647 		/*
648 		 * Remove the problematic req upon fatal errors on the server
649 		 */
650 		if (nfs_error_is_fatal(ret)) {
651 			nfs_context_set_write_error(req->wb_context, ret);
652 			if (nfs_error_is_fatal_on_server(ret))
653 				goto out_launder;
654 		} else
655 			ret = -EAGAIN;
656 		nfs_redirty_request(req);
657 	} else
658 		nfs_add_stats(page_file_mapping(page)->host,
659 				NFSIOS_WRITEPAGES, 1);
660 out:
661 	return ret;
662 out_launder:
663 	nfs_write_error_remove_page(req);
664 	return 0;
665 }
666 
nfs_do_writepage(struct page * page,struct writeback_control * wbc,struct nfs_pageio_descriptor * pgio)667 static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
668 			    struct nfs_pageio_descriptor *pgio)
669 {
670 	int ret;
671 
672 	nfs_pageio_cond_complete(pgio, page_index(page));
673 	ret = nfs_page_async_flush(pgio, page);
674 	if (ret == -EAGAIN) {
675 		redirty_page_for_writepage(wbc, page);
676 		ret = 0;
677 	}
678 	return ret;
679 }
680 
681 /*
682  * Write an mmapped page to the server.
683  */
nfs_writepage_locked(struct page * page,struct writeback_control * wbc)684 static int nfs_writepage_locked(struct page *page,
685 				struct writeback_control *wbc)
686 {
687 	struct nfs_pageio_descriptor pgio;
688 	struct inode *inode = page_file_mapping(page)->host;
689 	int err;
690 
691 	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
692 	nfs_pageio_init_write(&pgio, inode, 0,
693 				false, &nfs_async_write_completion_ops);
694 	err = nfs_do_writepage(page, wbc, &pgio);
695 	nfs_pageio_complete(&pgio);
696 	if (err < 0)
697 		return err;
698 	if (pgio.pg_error < 0)
699 		return pgio.pg_error;
700 	return 0;
701 }
702 
nfs_writepage(struct page * page,struct writeback_control * wbc)703 int nfs_writepage(struct page *page, struct writeback_control *wbc)
704 {
705 	int ret;
706 
707 	ret = nfs_writepage_locked(page, wbc);
708 	unlock_page(page);
709 	return ret;
710 }
711 
nfs_writepages_callback(struct page * page,struct writeback_control * wbc,void * data)712 static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
713 {
714 	int ret;
715 
716 	ret = nfs_do_writepage(page, wbc, data);
717 	unlock_page(page);
718 	return ret;
719 }
720 
nfs_io_completion_commit(void * inode)721 static void nfs_io_completion_commit(void *inode)
722 {
723 	nfs_commit_inode(inode, 0);
724 }
725 
nfs_writepages(struct address_space * mapping,struct writeback_control * wbc)726 int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
727 {
728 	struct inode *inode = mapping->host;
729 	struct nfs_pageio_descriptor pgio;
730 	struct nfs_io_completion *ioc = nfs_io_completion_alloc(GFP_NOFS);
731 	int err;
732 
733 	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
734 
735 	if (ioc)
736 		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);
737 
738 	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
739 				&nfs_async_write_completion_ops);
740 	pgio.pg_io_completion = ioc;
741 	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
742 	nfs_pageio_complete(&pgio);
743 	nfs_io_completion_put(ioc);
744 
745 	if (err < 0)
746 		goto out_err;
747 	err = pgio.pg_error;
748 	if (err < 0)
749 		goto out_err;
750 	return 0;
751 out_err:
752 	return err;
753 }
754 
755 /*
756  * Insert a write request into an inode
757  */
nfs_inode_add_request(struct inode * inode,struct nfs_page * req)758 static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
759 {
760 	struct address_space *mapping = page_file_mapping(req->wb_page);
761 	struct nfs_inode *nfsi = NFS_I(inode);
762 
763 	WARN_ON_ONCE(req->wb_this_page != req);
764 
765 	/* Lock the request! */
766 	nfs_lock_request(req);
767 
768 	/*
769 	 * Swap-space should not get truncated. Hence no need to plug the race
770 	 * with invalidate/truncate.
771 	 */
772 	spin_lock(&mapping->private_lock);
773 	if (!nfs_have_writebacks(inode) &&
774 	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
775 		inode_inc_iversion_raw(inode);
776 	if (likely(!PageSwapCache(req->wb_page))) {
777 		set_bit(PG_MAPPED, &req->wb_flags);
778 		SetPagePrivate(req->wb_page);
779 		set_page_private(req->wb_page, (unsigned long)req);
780 	}
781 	spin_unlock(&mapping->private_lock);
782 	atomic_long_inc(&nfsi->nrequests);
783 	/* this a head request for a page group - mark it as having an
784 	 * extra reference so sub groups can follow suit.
785 	 * This flag also informs pgio layer when to bump nrequests when
786 	 * adding subrequests. */
787 	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
788 	kref_get(&req->wb_kref);
789 }
790 
791 /*
792  * Remove a write request from an inode
793  */
nfs_inode_remove_request(struct nfs_page * req)794 static void nfs_inode_remove_request(struct nfs_page *req)
795 {
796 	struct address_space *mapping = page_file_mapping(req->wb_page);
797 	struct inode *inode = mapping->host;
798 	struct nfs_inode *nfsi = NFS_I(inode);
799 	struct nfs_page *head;
800 
801 	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
802 		head = req->wb_head;
803 
804 		spin_lock(&mapping->private_lock);
805 		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
806 			set_page_private(head->wb_page, 0);
807 			ClearPagePrivate(head->wb_page);
808 			clear_bit(PG_MAPPED, &head->wb_flags);
809 		}
810 		spin_unlock(&mapping->private_lock);
811 	}
812 
813 	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) {
814 		nfs_release_request(req);
815 		atomic_long_dec(&nfsi->nrequests);
816 	}
817 }
818 
819 static void
nfs_mark_request_dirty(struct nfs_page * req)820 nfs_mark_request_dirty(struct nfs_page *req)
821 {
822 	if (req->wb_page)
823 		__set_page_dirty_nobuffers(req->wb_page);
824 }
825 
826 /*
827  * nfs_page_search_commits_for_head_request_locked
828  *
829  * Search through commit lists on @inode for the head request for @page.
830  * Must be called while holding the inode (which is cinfo) lock.
831  *
832  * Returns the head request if found, or NULL if not found.
833  */
834 static struct nfs_page *
nfs_page_search_commits_for_head_request_locked(struct nfs_inode * nfsi,struct page * page)835 nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
836 						struct page *page)
837 {
838 	struct nfs_page *freq, *t;
839 	struct nfs_commit_info cinfo;
840 	struct inode *inode = &nfsi->vfs_inode;
841 
842 	nfs_init_cinfo_from_inode(&cinfo, inode);
843 
844 	/* search through pnfs commit lists */
845 	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
846 	if (freq)
847 		return freq->wb_head;
848 
849 	/* Linearly search the commit list for the correct request */
850 	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
851 		if (freq->wb_page == page)
852 			return freq->wb_head;
853 	}
854 
855 	return NULL;
856 }
857 
858 /**
859  * nfs_request_add_commit_list_locked - add request to a commit list
860  * @req: pointer to a struct nfs_page
861  * @dst: commit list head
862  * @cinfo: holds list lock and accounting info
863  *
864  * This sets the PG_CLEAN bit, updates the cinfo count of
865  * number of outstanding requests requiring a commit as well as
866  * the MM page stats.
867  *
868  * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
869  * nfs_page lock.
870  */
871 void
nfs_request_add_commit_list_locked(struct nfs_page * req,struct list_head * dst,struct nfs_commit_info * cinfo)872 nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
873 			    struct nfs_commit_info *cinfo)
874 {
875 	set_bit(PG_CLEAN, &req->wb_flags);
876 	nfs_list_add_request(req, dst);
877 	atomic_long_inc(&cinfo->mds->ncommit);
878 }
879 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);
880 
881 /**
882  * nfs_request_add_commit_list - add request to a commit list
883  * @req: pointer to a struct nfs_page
884  * @dst: commit list head
885  * @cinfo: holds list lock and accounting info
886  *
887  * This sets the PG_CLEAN bit, updates the cinfo count of
888  * number of outstanding requests requiring a commit as well as
889  * the MM page stats.
890  *
891  * The caller must _not_ hold the cinfo->lock, but must be
892  * holding the nfs_page lock.
893  */
894 void
nfs_request_add_commit_list(struct nfs_page * req,struct nfs_commit_info * cinfo)895 nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
896 {
897 	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
898 	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
899 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
900 	if (req->wb_page)
901 		nfs_mark_page_unstable(req->wb_page, cinfo);
902 }
903 EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
904 
905 /**
906  * nfs_request_remove_commit_list - Remove request from a commit list
907  * @req: pointer to a nfs_page
908  * @cinfo: holds list lock and accounting info
909  *
910  * This clears the PG_CLEAN bit, and updates the cinfo's count of
911  * number of outstanding requests requiring a commit
912  * It does not update the MM page stats.
913  *
914  * The caller _must_ hold the cinfo->lock and the nfs_page lock.
915  */
916 void
nfs_request_remove_commit_list(struct nfs_page * req,struct nfs_commit_info * cinfo)917 nfs_request_remove_commit_list(struct nfs_page *req,
918 			       struct nfs_commit_info *cinfo)
919 {
920 	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
921 		return;
922 	nfs_list_remove_request(req);
923 	atomic_long_dec(&cinfo->mds->ncommit);
924 }
925 EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
926 
nfs_init_cinfo_from_inode(struct nfs_commit_info * cinfo,struct inode * inode)927 static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
928 				      struct inode *inode)
929 {
930 	cinfo->inode = inode;
931 	cinfo->mds = &NFS_I(inode)->commit_info;
932 	cinfo->ds = pnfs_get_ds_info(inode);
933 	cinfo->dreq = NULL;
934 	cinfo->completion_ops = &nfs_commit_completion_ops;
935 }
936 
nfs_init_cinfo(struct nfs_commit_info * cinfo,struct inode * inode,struct nfs_direct_req * dreq)937 void nfs_init_cinfo(struct nfs_commit_info *cinfo,
938 		    struct inode *inode,
939 		    struct nfs_direct_req *dreq)
940 {
941 	if (dreq)
942 		nfs_init_cinfo_from_dreq(cinfo, dreq);
943 	else
944 		nfs_init_cinfo_from_inode(cinfo, inode);
945 }
946 EXPORT_SYMBOL_GPL(nfs_init_cinfo);
947 
948 /*
949  * Add a request to the inode's commit list.
950  */
951 void
nfs_mark_request_commit(struct nfs_page * req,struct pnfs_layout_segment * lseg,struct nfs_commit_info * cinfo,u32 ds_commit_idx)952 nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
953 			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
954 {
955 	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
956 		return;
957 	nfs_request_add_commit_list(req, cinfo);
958 }
959 
960 static void
nfs_clear_page_commit(struct page * page)961 nfs_clear_page_commit(struct page *page)
962 {
963 	dec_node_page_state(page, NR_UNSTABLE_NFS);
964 	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
965 		    WB_RECLAIMABLE);
966 }
967 
968 /* Called holding the request lock on @req */
969 static void
nfs_clear_request_commit(struct nfs_page * req)970 nfs_clear_request_commit(struct nfs_page *req)
971 {
972 	if (test_bit(PG_CLEAN, &req->wb_flags)) {
973 		struct inode *inode = d_inode(req->wb_context->dentry);
974 		struct nfs_commit_info cinfo;
975 
976 		nfs_init_cinfo_from_inode(&cinfo, inode);
977 		mutex_lock(&NFS_I(inode)->commit_mutex);
978 		if (!pnfs_clear_request_commit(req, &cinfo)) {
979 			nfs_request_remove_commit_list(req, &cinfo);
980 		}
981 		mutex_unlock(&NFS_I(inode)->commit_mutex);
982 		nfs_clear_page_commit(req->wb_page);
983 	}
984 }
985 
nfs_write_need_commit(struct nfs_pgio_header * hdr)986 int nfs_write_need_commit(struct nfs_pgio_header *hdr)
987 {
988 	if (hdr->verf.committed == NFS_DATA_SYNC)
989 		return hdr->lseg == NULL;
990 	return hdr->verf.committed != NFS_FILE_SYNC;
991 }
992 
nfs_async_write_init(struct nfs_pgio_header * hdr)993 static void nfs_async_write_init(struct nfs_pgio_header *hdr)
994 {
995 	nfs_io_completion_get(hdr->io_completion);
996 }
997 
nfs_write_completion(struct nfs_pgio_header * hdr)998 static void nfs_write_completion(struct nfs_pgio_header *hdr)
999 {
1000 	struct nfs_commit_info cinfo;
1001 	unsigned long bytes = 0;
1002 
1003 	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
1004 		goto out;
1005 	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
1006 	while (!list_empty(&hdr->pages)) {
1007 		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
1008 
1009 		bytes += req->wb_bytes;
1010 		nfs_list_remove_request(req);
1011 		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
1012 		    (hdr->good_bytes < bytes)) {
1013 			nfs_set_pageerror(page_file_mapping(req->wb_page));
1014 			nfs_context_set_write_error(req->wb_context, hdr->error);
1015 			goto remove_req;
1016 		}
1017 		if (nfs_write_need_commit(hdr)) {
1018 			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1019 			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1020 				hdr->pgio_mirror_idx);
1021 			goto next;
1022 		}
1023 remove_req:
1024 		nfs_inode_remove_request(req);
1025 next:
1026 		nfs_end_page_writeback(req);
1027 		nfs_release_request(req);
1028 	}
1029 out:
1030 	nfs_io_completion_put(hdr->io_completion);
1031 	hdr->release(hdr);
1032 }
1033 
1034 unsigned long
nfs_reqs_to_commit(struct nfs_commit_info * cinfo)1035 nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1036 {
1037 	return atomic_long_read(&cinfo->mds->ncommit);
1038 }
1039 
1040 /* NFS_I(cinfo->inode)->commit_mutex held by caller */
1041 int
nfs_scan_commit_list(struct list_head * src,struct list_head * dst,struct nfs_commit_info * cinfo,int max)1042 nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
1043 		     struct nfs_commit_info *cinfo, int max)
1044 {
1045 	struct nfs_page *req, *tmp;
1046 	int ret = 0;
1047 
1048 	list_for_each_entry_safe(req, tmp, src, wb_list) {
1049 		kref_get(&req->wb_kref);
1050 		if (!nfs_lock_request(req)) {
1051 			nfs_release_request(req);
1052 			continue;
1053 		}
1054 		nfs_request_remove_commit_list(req, cinfo);
1055 		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1056 		nfs_list_add_request(req, dst);
1057 		ret++;
1058 		if ((ret == max) && !cinfo->dreq)
1059 			break;
1060 		cond_resched();
1061 	}
1062 	return ret;
1063 }
1064 EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1065 
1066 /*
1067  * nfs_scan_commit - Scan an inode for commit requests
1068  * @inode: NFS inode to scan
1069  * @dst: mds destination list
1070  * @cinfo: mds and ds lists of reqs ready to commit
1071  *
1072  * Moves requests from the inode's 'commit' request list.
1073  * The requests are *not* checked to ensure that they form a contiguous set.
1074  */
1075 int
nfs_scan_commit(struct inode * inode,struct list_head * dst,struct nfs_commit_info * cinfo)1076 nfs_scan_commit(struct inode *inode, struct list_head *dst,
1077 		struct nfs_commit_info *cinfo)
1078 {
1079 	int ret = 0;
1080 
1081 	if (!atomic_long_read(&cinfo->mds->ncommit))
1082 		return 0;
1083 	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1084 	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1085 		const int max = INT_MAX;
1086 
1087 		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
1088 					   cinfo, max);
1089 		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1090 	}
1091 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1092 	return ret;
1093 }
1094 
1095 /*
1096  * Search for an existing write request, and attempt to update
1097  * it to reflect a new dirty region on a given page.
1098  *
1099  * If the attempt fails, then the existing request is flushed out
1100  * to disk.
1101  */
nfs_try_to_update_request(struct inode * inode,struct page * page,unsigned int offset,unsigned int bytes)1102 static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
1103 		struct page *page,
1104 		unsigned int offset,
1105 		unsigned int bytes)
1106 {
1107 	struct nfs_page *req;
1108 	unsigned int rqend;
1109 	unsigned int end;
1110 	int error;
1111 
1112 	end = offset + bytes;
1113 
1114 	req = nfs_lock_and_join_requests(page);
1115 	if (IS_ERR_OR_NULL(req))
1116 		return req;
1117 
1118 	rqend = req->wb_offset + req->wb_bytes;
1119 	/*
1120 	 * Tell the caller to flush out the request if
1121 	 * the offsets are non-contiguous.
1122 	 * Note: nfs_flush_incompatible() will already
1123 	 * have flushed out requests having wrong owners.
1124 	 */
1125 	if (offset > rqend || end < req->wb_offset)
1126 		goto out_flushme;
1127 
1128 	/* Okay, the request matches. Update the region */
1129 	if (offset < req->wb_offset) {
1130 		req->wb_offset = offset;
1131 		req->wb_pgbase = offset;
1132 	}
1133 	if (end > rqend)
1134 		req->wb_bytes = end - req->wb_offset;
1135 	else
1136 		req->wb_bytes = rqend - req->wb_offset;
1137 	return req;
1138 out_flushme:
1139 	/*
1140 	 * Note: we mark the request dirty here because
1141 	 * nfs_lock_and_join_requests() cannot preserve
1142 	 * commit flags, so we have to replay the write.
1143 	 */
1144 	nfs_mark_request_dirty(req);
1145 	nfs_unlock_and_release_request(req);
1146 	error = nfs_wb_page(inode, page);
1147 	return (error < 0) ? ERR_PTR(error) : NULL;
1148 }
1149 
1150 /*
1151  * Try to update an existing write request, or create one if there is none.
1152  *
1153  * Note: Should always be called with the Page Lock held to prevent races
1154  * if we have to add a new request. Also assumes that the caller has
1155  * already called nfs_flush_incompatible() if necessary.
1156  */
nfs_setup_write_request(struct nfs_open_context * ctx,struct page * page,unsigned int offset,unsigned int bytes)1157 static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
1158 		struct page *page, unsigned int offset, unsigned int bytes)
1159 {
1160 	struct inode *inode = page_file_mapping(page)->host;
1161 	struct nfs_page	*req;
1162 
1163 	req = nfs_try_to_update_request(inode, page, offset, bytes);
1164 	if (req != NULL)
1165 		goto out;
1166 	req = nfs_create_request(ctx, page, NULL, offset, bytes);
1167 	if (IS_ERR(req))
1168 		goto out;
1169 	nfs_inode_add_request(inode, req);
1170 out:
1171 	return req;
1172 }
1173 
nfs_writepage_setup(struct nfs_open_context * ctx,struct page * page,unsigned int offset,unsigned int count)1174 static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
1175 		unsigned int offset, unsigned int count)
1176 {
1177 	struct nfs_page	*req;
1178 
1179 	req = nfs_setup_write_request(ctx, page, offset, count);
1180 	if (IS_ERR(req))
1181 		return PTR_ERR(req);
1182 	/* Update file length */
1183 	nfs_grow_file(page, offset, count);
1184 	nfs_mark_uptodate(req);
1185 	nfs_mark_request_dirty(req);
1186 	nfs_unlock_and_release_request(req);
1187 	return 0;
1188 }
1189 
nfs_flush_incompatible(struct file * file,struct page * page)1190 int nfs_flush_incompatible(struct file *file, struct page *page)
1191 {
1192 	struct nfs_open_context *ctx = nfs_file_open_context(file);
1193 	struct nfs_lock_context *l_ctx;
1194 	struct file_lock_context *flctx = file_inode(file)->i_flctx;
1195 	struct nfs_page	*req;
1196 	int do_flush, status;
1197 	/*
1198 	 * Look for a request corresponding to this page. If there
1199 	 * is one, and it belongs to another file, we flush it out
1200 	 * before we try to copy anything into the page. Do this
1201 	 * due to the lack of an ACCESS-type call in NFSv2.
1202 	 * Also do the same if we find a request from an existing
1203 	 * dropped page.
1204 	 */
1205 	do {
1206 		req = nfs_page_find_head_request(page);
1207 		if (req == NULL)
1208 			return 0;
1209 		l_ctx = req->wb_lock_context;
1210 		do_flush = req->wb_page != page ||
1211 			!nfs_match_open_context(req->wb_context, ctx);
1212 		if (l_ctx && flctx &&
1213 		    !(list_empty_careful(&flctx->flc_posix) &&
1214 		      list_empty_careful(&flctx->flc_flock))) {
1215 			do_flush |= l_ctx->lockowner != current->files;
1216 		}
1217 		nfs_release_request(req);
1218 		if (!do_flush)
1219 			return 0;
1220 		status = nfs_wb_page(page_file_mapping(page)->host, page);
1221 	} while (status == 0);
1222 	return status;
1223 }
1224 
1225 /*
1226  * Avoid buffered writes when a open context credential's key would
1227  * expire soon.
1228  *
1229  * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
1230  *
1231  * Return 0 and set a credential flag which triggers the inode to flush
1232  * and performs  NFS_FILE_SYNC writes if the key will expired within
1233  * RPC_KEY_EXPIRE_TIMEO.
1234  */
1235 int
nfs_key_timeout_notify(struct file * filp,struct inode * inode)1236 nfs_key_timeout_notify(struct file *filp, struct inode *inode)
1237 {
1238 	struct nfs_open_context *ctx = nfs_file_open_context(filp);
1239 	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1240 
1241 	return rpcauth_key_timeout_notify(auth, ctx->cred);
1242 }
1243 
1244 /*
1245  * Test if the open context credential key is marked to expire soon.
1246  */
nfs_ctx_key_to_expire(struct nfs_open_context * ctx,struct inode * inode)1247 bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1248 {
1249 	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1250 
1251 	return rpcauth_cred_key_to_expire(auth, ctx->cred);
1252 }
1253 
1254 /*
1255  * If the page cache is marked as unsafe or invalid, then we can't rely on
1256  * the PageUptodate() flag. In this case, we will need to turn off
1257  * write optimisations that depend on the page contents being correct.
1258  */
nfs_write_pageuptodate(struct page * page,struct inode * inode)1259 static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1260 {
1261 	struct nfs_inode *nfsi = NFS_I(inode);
1262 
1263 	if (nfs_have_delegated_attributes(inode))
1264 		goto out;
1265 	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1266 		return false;
1267 	smp_rmb();
1268 	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1269 		return false;
1270 out:
1271 	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
1272 		return false;
1273 	return PageUptodate(page) != 0;
1274 }
1275 
1276 static bool
is_whole_file_wrlock(struct file_lock * fl)1277 is_whole_file_wrlock(struct file_lock *fl)
1278 {
1279 	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
1280 			fl->fl_type == F_WRLCK;
1281 }
1282 
1283 /* If we know the page is up to date, and we're not using byte range locks (or
1284  * if we have the whole file locked for writing), it may be more efficient to
1285  * extend the write to cover the entire page in order to avoid fragmentation
1286  * inefficiencies.
1287  *
1288  * If the file is opened for synchronous writes then we can just skip the rest
1289  * of the checks.
1290  */
nfs_can_extend_write(struct file * file,struct page * page,struct inode * inode)1291 static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
1292 {
1293 	int ret;
1294 	struct file_lock_context *flctx = inode->i_flctx;
1295 	struct file_lock *fl;
1296 
1297 	if (file->f_flags & O_DSYNC)
1298 		return 0;
1299 	if (!nfs_write_pageuptodate(page, inode))
1300 		return 0;
1301 	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
1302 		return 1;
1303 	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
1304 		       list_empty_careful(&flctx->flc_posix)))
1305 		return 1;
1306 
1307 	/* Check to see if there are whole file write locks */
1308 	ret = 0;
1309 	spin_lock(&flctx->flc_lock);
1310 	if (!list_empty(&flctx->flc_posix)) {
1311 		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
1312 					fl_list);
1313 		if (is_whole_file_wrlock(fl))
1314 			ret = 1;
1315 	} else if (!list_empty(&flctx->flc_flock)) {
1316 		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
1317 					fl_list);
1318 		if (fl->fl_type == F_WRLCK)
1319 			ret = 1;
1320 	}
1321 	spin_unlock(&flctx->flc_lock);
1322 	return ret;
1323 }
1324 
1325 /*
1326  * Update and possibly write a cached page of an NFS file.
1327  *
1328  * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
1329  * things with a page scheduled for an RPC call (e.g. invalidate it).
1330  */
nfs_updatepage(struct file * file,struct page * page,unsigned int offset,unsigned int count)1331 int nfs_updatepage(struct file *file, struct page *page,
1332 		unsigned int offset, unsigned int count)
1333 {
1334 	struct nfs_open_context *ctx = nfs_file_open_context(file);
1335 	struct address_space *mapping = page_file_mapping(page);
1336 	struct inode	*inode = mapping->host;
1337 	int		status = 0;
1338 
1339 	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
1340 
1341 	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
1342 		file, count, (long long)(page_file_offset(page) + offset));
1343 
1344 	if (!count)
1345 		goto out;
1346 
1347 	if (nfs_can_extend_write(file, page, inode)) {
1348 		count = max(count + offset, nfs_page_length(page));
1349 		offset = 0;
1350 	}
1351 
1352 	status = nfs_writepage_setup(ctx, page, offset, count);
1353 	if (status < 0)
1354 		nfs_set_pageerror(mapping);
1355 	else
1356 		__set_page_dirty_nobuffers(page);
1357 out:
1358 	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
1359 			status, (long long)i_size_read(inode));
1360 	return status;
1361 }
1362 
flush_task_priority(int how)1363 static int flush_task_priority(int how)
1364 {
1365 	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
1366 		case FLUSH_HIGHPRI:
1367 			return RPC_PRIORITY_HIGH;
1368 		case FLUSH_LOWPRI:
1369 			return RPC_PRIORITY_LOW;
1370 	}
1371 	return RPC_PRIORITY_NORMAL;
1372 }
1373 
nfs_initiate_write(struct nfs_pgio_header * hdr,struct rpc_message * msg,const struct nfs_rpc_ops * rpc_ops,struct rpc_task_setup * task_setup_data,int how)1374 static void nfs_initiate_write(struct nfs_pgio_header *hdr,
1375 			       struct rpc_message *msg,
1376 			       const struct nfs_rpc_ops *rpc_ops,
1377 			       struct rpc_task_setup *task_setup_data, int how)
1378 {
1379 	int priority = flush_task_priority(how);
1380 
1381 	task_setup_data->priority = priority;
1382 	rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
1383 	trace_nfs_initiate_write(hdr->inode, hdr->io_start, hdr->good_bytes,
1384 				 hdr->args.stable);
1385 }
1386 
1387 /* If a nfs_flush_* function fails, it should remove reqs from @head and
1388  * call this on each, which will prepare them to be retried on next
1389  * writeback using standard nfs.
1390  */
nfs_redirty_request(struct nfs_page * req)1391 static void nfs_redirty_request(struct nfs_page *req)
1392 {
1393 	nfs_mark_request_dirty(req);
1394 	set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1395 	nfs_end_page_writeback(req);
1396 	nfs_release_request(req);
1397 }
1398 
nfs_async_write_error(struct list_head * head,int error)1399 static void nfs_async_write_error(struct list_head *head, int error)
1400 {
1401 	struct nfs_page	*req;
1402 
1403 	while (!list_empty(head)) {
1404 		req = nfs_list_entry(head->next);
1405 		nfs_list_remove_request(req);
1406 		if (nfs_error_is_fatal(error)) {
1407 			nfs_context_set_write_error(req->wb_context, error);
1408 			if (nfs_error_is_fatal_on_server(error)) {
1409 				nfs_write_error_remove_page(req);
1410 				continue;
1411 			}
1412 		}
1413 		nfs_redirty_request(req);
1414 	}
1415 }
1416 
nfs_async_write_reschedule_io(struct nfs_pgio_header * hdr)1417 static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
1418 {
1419 	nfs_async_write_error(&hdr->pages, 0);
1420 	filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset,
1421 			hdr->args.offset + hdr->args.count - 1);
1422 }
1423 
1424 static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1425 	.init_hdr = nfs_async_write_init,
1426 	.error_cleanup = nfs_async_write_error,
1427 	.completion = nfs_write_completion,
1428 	.reschedule_io = nfs_async_write_reschedule_io,
1429 };
1430 
nfs_pageio_init_write(struct nfs_pageio_descriptor * pgio,struct inode * inode,int ioflags,bool force_mds,const struct nfs_pgio_completion_ops * compl_ops)1431 void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1432 			       struct inode *inode, int ioflags, bool force_mds,
1433 			       const struct nfs_pgio_completion_ops *compl_ops)
1434 {
1435 	struct nfs_server *server = NFS_SERVER(inode);
1436 	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1437 
1438 #ifdef CONFIG_NFS_V4_1
1439 	if (server->pnfs_curr_ld && !force_mds)
1440 		pg_ops = server->pnfs_curr_ld->pg_write_ops;
1441 #endif
1442 	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1443 			server->wsize, ioflags);
1444 }
1445 EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1446 
nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor * pgio)1447 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1448 {
1449 	struct nfs_pgio_mirror *mirror;
1450 
1451 	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
1452 		pgio->pg_ops->pg_cleanup(pgio);
1453 
1454 	pgio->pg_ops = &nfs_pgio_rw_ops;
1455 
1456 	nfs_pageio_stop_mirroring(pgio);
1457 
1458 	mirror = &pgio->pg_mirrors[0];
1459 	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1460 }
1461 EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1462 
1463 
nfs_commit_prepare(struct rpc_task * task,void * calldata)1464 void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1465 {
1466 	struct nfs_commit_data *data = calldata;
1467 
1468 	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1469 }
1470 
1471 /*
1472  * Special version of should_remove_suid() that ignores capabilities.
1473  */
nfs_should_remove_suid(const struct inode * inode)1474 static int nfs_should_remove_suid(const struct inode *inode)
1475 {
1476 	umode_t mode = inode->i_mode;
1477 	int kill = 0;
1478 
1479 	/* suid always must be killed */
1480 	if (unlikely(mode & S_ISUID))
1481 		kill = ATTR_KILL_SUID;
1482 
1483 	/*
1484 	 * sgid without any exec bits is just a mandatory locking mark; leave
1485 	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
1486 	 */
1487 	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1488 		kill |= ATTR_KILL_SGID;
1489 
1490 	if (unlikely(kill && S_ISREG(mode)))
1491 		return kill;
1492 
1493 	return 0;
1494 }
1495 
nfs_writeback_check_extend(struct nfs_pgio_header * hdr,struct nfs_fattr * fattr)1496 static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
1497 		struct nfs_fattr *fattr)
1498 {
1499 	struct nfs_pgio_args *argp = &hdr->args;
1500 	struct nfs_pgio_res *resp = &hdr->res;
1501 	u64 size = argp->offset + resp->count;
1502 
1503 	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1504 		fattr->size = size;
1505 	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
1506 		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1507 		return;
1508 	}
1509 	if (size != fattr->size)
1510 		return;
1511 	/* Set attribute barrier */
1512 	nfs_fattr_set_barrier(fattr);
1513 	/* ...and update size */
1514 	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1515 }
1516 
nfs_writeback_update_inode(struct nfs_pgio_header * hdr)1517 void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
1518 {
1519 	struct nfs_fattr *fattr = &hdr->fattr;
1520 	struct inode *inode = hdr->inode;
1521 
1522 	spin_lock(&inode->i_lock);
1523 	nfs_writeback_check_extend(hdr, fattr);
1524 	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
1525 	spin_unlock(&inode->i_lock);
1526 }
1527 EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);
1528 
1529 /*
1530  * This function is called when the WRITE call is complete.
1531  */
nfs_writeback_done(struct rpc_task * task,struct nfs_pgio_header * hdr,struct inode * inode)1532 static int nfs_writeback_done(struct rpc_task *task,
1533 			      struct nfs_pgio_header *hdr,
1534 			      struct inode *inode)
1535 {
1536 	int status;
1537 
1538 	/*
1539 	 * ->write_done will attempt to use post-op attributes to detect
1540 	 * conflicting writes by other clients.  A strict interpretation
1541 	 * of close-to-open would allow us to continue caching even if
1542 	 * another writer had changed the file, but some applications
1543 	 * depend on tighter cache coherency when writing.
1544 	 */
1545 	status = NFS_PROTO(inode)->write_done(task, hdr);
1546 	if (status != 0)
1547 		return status;
1548 
1549 	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1550 	trace_nfs_writeback_done(inode, task->tk_status,
1551 				 hdr->args.offset, hdr->res.verf);
1552 
1553 	if (hdr->res.verf->committed < hdr->args.stable &&
1554 	    task->tk_status >= 0) {
1555 		/* We tried a write call, but the server did not
1556 		 * commit data to stable storage even though we
1557 		 * requested it.
1558 		 * Note: There is a known bug in Tru64 < 5.0 in which
1559 		 *	 the server reports NFS_DATA_SYNC, but performs
1560 		 *	 NFS_FILE_SYNC. We therefore implement this checking
1561 		 *	 as a dprintk() in order to avoid filling syslog.
1562 		 */
1563 		static unsigned long    complain;
1564 
1565 		/* Note this will print the MDS for a DS write */
1566 		if (time_before(complain, jiffies)) {
1567 			dprintk("NFS:       faulty NFS server %s:"
1568 				" (committed = %d) != (stable = %d)\n",
1569 				NFS_SERVER(inode)->nfs_client->cl_hostname,
1570 				hdr->res.verf->committed, hdr->args.stable);
1571 			complain = jiffies + 300 * HZ;
1572 		}
1573 	}
1574 
1575 	/* Deal with the suid/sgid bit corner case */
1576 	if (nfs_should_remove_suid(inode)) {
1577 		spin_lock(&inode->i_lock);
1578 		NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
1579 		spin_unlock(&inode->i_lock);
1580 	}
1581 	return 0;
1582 }
1583 
1584 /*
1585  * This function is called when the WRITE call is complete.
1586  */
nfs_writeback_result(struct rpc_task * task,struct nfs_pgio_header * hdr)1587 static void nfs_writeback_result(struct rpc_task *task,
1588 				 struct nfs_pgio_header *hdr)
1589 {
1590 	struct nfs_pgio_args	*argp = &hdr->args;
1591 	struct nfs_pgio_res	*resp = &hdr->res;
1592 
1593 	if (resp->count < argp->count) {
1594 		static unsigned long    complain;
1595 
1596 		/* This a short write! */
1597 		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1598 
1599 		/* Has the server at least made some progress? */
1600 		if (resp->count == 0) {
1601 			if (time_before(complain, jiffies)) {
1602 				printk(KERN_WARNING
1603 				       "NFS: Server wrote zero bytes, expected %u.\n",
1604 				       argp->count);
1605 				complain = jiffies + 300 * HZ;
1606 			}
1607 			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1608 			task->tk_status = -EIO;
1609 			return;
1610 		}
1611 
1612 		/* For non rpc-based layout drivers, retry-through-MDS */
1613 		if (!task->tk_ops) {
1614 			hdr->pnfs_error = -EAGAIN;
1615 			return;
1616 		}
1617 
1618 		/* Was this an NFSv2 write or an NFSv3 stable write? */
1619 		if (resp->verf->committed != NFS_UNSTABLE) {
1620 			/* Resend from where the server left off */
1621 			hdr->mds_offset += resp->count;
1622 			argp->offset += resp->count;
1623 			argp->pgbase += resp->count;
1624 			argp->count -= resp->count;
1625 		} else {
1626 			/* Resend as a stable write in order to avoid
1627 			 * headaches in the case of a server crash.
1628 			 */
1629 			argp->stable = NFS_FILE_SYNC;
1630 		}
1631 		rpc_restart_call_prepare(task);
1632 	}
1633 }
1634 
wait_on_commit(struct nfs_mds_commit_info * cinfo)1635 static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1636 {
1637 	return wait_var_event_killable(&cinfo->rpcs_out,
1638 				       !atomic_read(&cinfo->rpcs_out));
1639 }
1640 
nfs_commit_begin(struct nfs_mds_commit_info * cinfo)1641 static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
1642 {
1643 	atomic_inc(&cinfo->rpcs_out);
1644 }
1645 
nfs_commit_end(struct nfs_mds_commit_info * cinfo)1646 static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1647 {
1648 	if (atomic_dec_and_test(&cinfo->rpcs_out))
1649 		wake_up_var(&cinfo->rpcs_out);
1650 }
1651 
nfs_commitdata_release(struct nfs_commit_data * data)1652 void nfs_commitdata_release(struct nfs_commit_data *data)
1653 {
1654 	put_nfs_open_context(data->context);
1655 	nfs_commit_free(data);
1656 }
1657 EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1658 
nfs_initiate_commit(struct rpc_clnt * clnt,struct nfs_commit_data * data,const struct nfs_rpc_ops * nfs_ops,const struct rpc_call_ops * call_ops,int how,int flags)1659 int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1660 			const struct nfs_rpc_ops *nfs_ops,
1661 			const struct rpc_call_ops *call_ops,
1662 			int how, int flags)
1663 {
1664 	struct rpc_task *task;
1665 	int priority = flush_task_priority(how);
1666 	struct rpc_message msg = {
1667 		.rpc_argp = &data->args,
1668 		.rpc_resp = &data->res,
1669 		.rpc_cred = data->cred,
1670 	};
1671 	struct rpc_task_setup task_setup_data = {
1672 		.task = &data->task,
1673 		.rpc_client = clnt,
1674 		.rpc_message = &msg,
1675 		.callback_ops = call_ops,
1676 		.callback_data = data,
1677 		.workqueue = nfsiod_workqueue,
1678 		.flags = RPC_TASK_ASYNC | flags,
1679 		.priority = priority,
1680 	};
1681 	/* Set up the initial task struct.  */
1682 	nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
1683 	trace_nfs_initiate_commit(data);
1684 
1685 	dprintk("NFS: initiated commit call\n");
1686 
1687 	task = rpc_run_task(&task_setup_data);
1688 	if (IS_ERR(task))
1689 		return PTR_ERR(task);
1690 	if (how & FLUSH_SYNC)
1691 		rpc_wait_for_completion_task(task);
1692 	rpc_put_task(task);
1693 	return 0;
1694 }
1695 EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1696 
nfs_get_lwb(struct list_head * head)1697 static loff_t nfs_get_lwb(struct list_head *head)
1698 {
1699 	loff_t lwb = 0;
1700 	struct nfs_page *req;
1701 
1702 	list_for_each_entry(req, head, wb_list)
1703 		if (lwb < (req_offset(req) + req->wb_bytes))
1704 			lwb = req_offset(req) + req->wb_bytes;
1705 
1706 	return lwb;
1707 }
1708 
1709 /*
1710  * Set up the argument/result storage required for the RPC call.
1711  */
nfs_init_commit(struct nfs_commit_data * data,struct list_head * head,struct pnfs_layout_segment * lseg,struct nfs_commit_info * cinfo)1712 void nfs_init_commit(struct nfs_commit_data *data,
1713 		     struct list_head *head,
1714 		     struct pnfs_layout_segment *lseg,
1715 		     struct nfs_commit_info *cinfo)
1716 {
1717 	struct nfs_page *first = nfs_list_entry(head->next);
1718 	struct inode *inode = d_inode(first->wb_context->dentry);
1719 
1720 	/* Set up the RPC argument and reply structs
1721 	 * NB: take care not to mess about with data->commit et al. */
1722 
1723 	list_splice_init(head, &data->pages);
1724 
1725 	data->inode	  = inode;
1726 	data->cred	  = first->wb_context->cred;
1727 	data->lseg	  = lseg; /* reference transferred */
1728 	/* only set lwb for pnfs commit */
1729 	if (lseg)
1730 		data->lwb = nfs_get_lwb(&data->pages);
1731 	data->mds_ops     = &nfs_commit_ops;
1732 	data->completion_ops = cinfo->completion_ops;
1733 	data->dreq	  = cinfo->dreq;
1734 
1735 	data->args.fh     = NFS_FH(data->inode);
1736 	/* Note: we always request a commit of the entire inode */
1737 	data->args.offset = 0;
1738 	data->args.count  = 0;
1739 	data->context     = get_nfs_open_context(first->wb_context);
1740 	data->res.fattr   = &data->fattr;
1741 	data->res.verf    = &data->verf;
1742 	nfs_fattr_init(&data->fattr);
1743 }
1744 EXPORT_SYMBOL_GPL(nfs_init_commit);
1745 
nfs_retry_commit(struct list_head * page_list,struct pnfs_layout_segment * lseg,struct nfs_commit_info * cinfo,u32 ds_commit_idx)1746 void nfs_retry_commit(struct list_head *page_list,
1747 		      struct pnfs_layout_segment *lseg,
1748 		      struct nfs_commit_info *cinfo,
1749 		      u32 ds_commit_idx)
1750 {
1751 	struct nfs_page *req;
1752 
1753 	while (!list_empty(page_list)) {
1754 		req = nfs_list_entry(page_list->next);
1755 		nfs_list_remove_request(req);
1756 		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1757 		if (!cinfo->dreq)
1758 			nfs_clear_page_commit(req->wb_page);
1759 		nfs_unlock_and_release_request(req);
1760 	}
1761 }
1762 EXPORT_SYMBOL_GPL(nfs_retry_commit);
1763 
1764 static void
nfs_commit_resched_write(struct nfs_commit_info * cinfo,struct nfs_page * req)1765 nfs_commit_resched_write(struct nfs_commit_info *cinfo,
1766 		struct nfs_page *req)
1767 {
1768 	__set_page_dirty_nobuffers(req->wb_page);
1769 }
1770 
1771 /*
1772  * Commit dirty pages
1773  */
1774 static int
nfs_commit_list(struct inode * inode,struct list_head * head,int how,struct nfs_commit_info * cinfo)1775 nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1776 		struct nfs_commit_info *cinfo)
1777 {
1778 	struct nfs_commit_data	*data;
1779 
1780 	/* another commit raced with us */
1781 	if (list_empty(head))
1782 		return 0;
1783 
1784 	data = nfs_commitdata_alloc(true);
1785 
1786 	/* Set up the argument struct */
1787 	nfs_init_commit(data, head, NULL, cinfo);
1788 	atomic_inc(&cinfo->mds->rpcs_out);
1789 	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
1790 				   data->mds_ops, how, 0);
1791 }
1792 
1793 /*
1794  * COMMIT call returned
1795  */
nfs_commit_done(struct rpc_task * task,void * calldata)1796 static void nfs_commit_done(struct rpc_task *task, void *calldata)
1797 {
1798 	struct nfs_commit_data	*data = calldata;
1799 
1800         dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1801                                 task->tk_pid, task->tk_status);
1802 
1803 	/* Call the NFS version-specific code */
1804 	NFS_PROTO(data->inode)->commit_done(task, data);
1805 	trace_nfs_commit_done(data);
1806 }
1807 
nfs_commit_release_pages(struct nfs_commit_data * data)1808 static void nfs_commit_release_pages(struct nfs_commit_data *data)
1809 {
1810 	const struct nfs_writeverf *verf = data->res.verf;
1811 	struct nfs_page	*req;
1812 	int status = data->task.tk_status;
1813 	struct nfs_commit_info cinfo;
1814 	struct nfs_server *nfss;
1815 
1816 	while (!list_empty(&data->pages)) {
1817 		req = nfs_list_entry(data->pages.next);
1818 		nfs_list_remove_request(req);
1819 		if (req->wb_page)
1820 			nfs_clear_page_commit(req->wb_page);
1821 
1822 		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1823 			req->wb_context->dentry->d_sb->s_id,
1824 			(unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
1825 			req->wb_bytes,
1826 			(long long)req_offset(req));
1827 		if (status < 0) {
1828 			nfs_context_set_write_error(req->wb_context, status);
1829 			if (req->wb_page)
1830 				nfs_inode_remove_request(req);
1831 			dprintk_cont(", error = %d\n", status);
1832 			goto next;
1833 		}
1834 
1835 		/* Okay, COMMIT succeeded, apparently. Check the verifier
1836 		 * returned by the server against all stored verfs. */
1837 		if (verf->committed > NFS_UNSTABLE &&
1838 		    !nfs_write_verifier_cmp(&req->wb_verf, &verf->verifier)) {
1839 			/* We have a match */
1840 			if (req->wb_page)
1841 				nfs_inode_remove_request(req);
1842 			dprintk_cont(" OK\n");
1843 			goto next;
1844 		}
1845 		/* We have a mismatch. Write the page again */
1846 		dprintk_cont(" mismatch\n");
1847 		nfs_mark_request_dirty(req);
1848 		set_bit(NFS_CONTEXT_RESEND_WRITES, &req->wb_context->flags);
1849 	next:
1850 		nfs_unlock_and_release_request(req);
1851 		/* Latency breaker */
1852 		cond_resched();
1853 	}
1854 	nfss = NFS_SERVER(data->inode);
1855 	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1856 		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1857 
1858 	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1859 	nfs_commit_end(cinfo.mds);
1860 }
1861 
nfs_commit_release(void * calldata)1862 static void nfs_commit_release(void *calldata)
1863 {
1864 	struct nfs_commit_data *data = calldata;
1865 
1866 	data->completion_ops->completion(data);
1867 	nfs_commitdata_release(calldata);
1868 }
1869 
1870 static const struct rpc_call_ops nfs_commit_ops = {
1871 	.rpc_call_prepare = nfs_commit_prepare,
1872 	.rpc_call_done = nfs_commit_done,
1873 	.rpc_release = nfs_commit_release,
1874 };
1875 
1876 static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1877 	.completion = nfs_commit_release_pages,
1878 	.resched_write = nfs_commit_resched_write,
1879 };
1880 
nfs_generic_commit_list(struct inode * inode,struct list_head * head,int how,struct nfs_commit_info * cinfo)1881 int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1882 			    int how, struct nfs_commit_info *cinfo)
1883 {
1884 	int status;
1885 
1886 	status = pnfs_commit_list(inode, head, how, cinfo);
1887 	if (status == PNFS_NOT_ATTEMPTED)
1888 		status = nfs_commit_list(inode, head, how, cinfo);
1889 	return status;
1890 }
1891 
__nfs_commit_inode(struct inode * inode,int how,struct writeback_control * wbc)1892 static int __nfs_commit_inode(struct inode *inode, int how,
1893 		struct writeback_control *wbc)
1894 {
1895 	LIST_HEAD(head);
1896 	struct nfs_commit_info cinfo;
1897 	int may_wait = how & FLUSH_SYNC;
1898 	int ret, nscan;
1899 
1900 	how &= ~FLUSH_SYNC;
1901 	nfs_init_cinfo_from_inode(&cinfo, inode);
1902 	nfs_commit_begin(cinfo.mds);
1903 	for (;;) {
1904 		ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
1905 		if (ret <= 0)
1906 			break;
1907 		ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
1908 		if (ret < 0)
1909 			break;
1910 		ret = 0;
1911 		if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
1912 			if (nscan < wbc->nr_to_write)
1913 				wbc->nr_to_write -= nscan;
1914 			else
1915 				wbc->nr_to_write = 0;
1916 		}
1917 		if (nscan < INT_MAX)
1918 			break;
1919 		cond_resched();
1920 	}
1921 	nfs_commit_end(cinfo.mds);
1922 	if (ret || !may_wait)
1923 		return ret;
1924 	return wait_on_commit(cinfo.mds);
1925 }
1926 
nfs_commit_inode(struct inode * inode,int how)1927 int nfs_commit_inode(struct inode *inode, int how)
1928 {
1929 	return __nfs_commit_inode(inode, how, NULL);
1930 }
1931 EXPORT_SYMBOL_GPL(nfs_commit_inode);
1932 
nfs_write_inode(struct inode * inode,struct writeback_control * wbc)1933 int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1934 {
1935 	struct nfs_inode *nfsi = NFS_I(inode);
1936 	int flags = FLUSH_SYNC;
1937 	int ret = 0;
1938 
1939 	if (wbc->sync_mode == WB_SYNC_NONE) {
1940 		/* no commits means nothing needs to be done */
1941 		if (!atomic_long_read(&nfsi->commit_info.ncommit))
1942 			goto check_requests_outstanding;
1943 
1944 		/* Don't commit yet if this is a non-blocking flush and there
1945 		 * are a lot of outstanding writes for this mapping.
1946 		 */
1947 		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1948 			goto out_mark_dirty;
1949 
1950 		/* don't wait for the COMMIT response */
1951 		flags = 0;
1952 	}
1953 
1954 	ret = __nfs_commit_inode(inode, flags, wbc);
1955 	if (!ret) {
1956 		if (flags & FLUSH_SYNC)
1957 			return 0;
1958 	} else if (atomic_long_read(&nfsi->commit_info.ncommit))
1959 		goto out_mark_dirty;
1960 
1961 check_requests_outstanding:
1962 	if (!atomic_read(&nfsi->commit_info.rpcs_out))
1963 		return ret;
1964 out_mark_dirty:
1965 	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1966 	return ret;
1967 }
1968 EXPORT_SYMBOL_GPL(nfs_write_inode);
1969 
1970 /*
1971  * Wrapper for filemap_write_and_wait_range()
1972  *
1973  * Needed for pNFS in order to ensure data becomes visible to the
1974  * client.
1975  */
nfs_filemap_write_and_wait_range(struct address_space * mapping,loff_t lstart,loff_t lend)1976 int nfs_filemap_write_and_wait_range(struct address_space *mapping,
1977 		loff_t lstart, loff_t lend)
1978 {
1979 	int ret;
1980 
1981 	ret = filemap_write_and_wait_range(mapping, lstart, lend);
1982 	if (ret == 0)
1983 		ret = pnfs_sync_inode(mapping->host, true);
1984 	return ret;
1985 }
1986 EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);
1987 
1988 /*
1989  * flush the inode to disk.
1990  */
nfs_wb_all(struct inode * inode)1991 int nfs_wb_all(struct inode *inode)
1992 {
1993 	int ret;
1994 
1995 	trace_nfs_writeback_inode_enter(inode);
1996 
1997 	ret = filemap_write_and_wait(inode->i_mapping);
1998 	if (ret)
1999 		goto out;
2000 	ret = nfs_commit_inode(inode, FLUSH_SYNC);
2001 	if (ret < 0)
2002 		goto out;
2003 	pnfs_sync_inode(inode, true);
2004 	ret = 0;
2005 
2006 out:
2007 	trace_nfs_writeback_inode_exit(inode, ret);
2008 	return ret;
2009 }
2010 EXPORT_SYMBOL_GPL(nfs_wb_all);
2011 
nfs_wb_page_cancel(struct inode * inode,struct page * page)2012 int nfs_wb_page_cancel(struct inode *inode, struct page *page)
2013 {
2014 	struct nfs_page *req;
2015 	int ret = 0;
2016 
2017 	wait_on_page_writeback(page);
2018 
2019 	/* blocking call to cancel all requests and join to a single (head)
2020 	 * request */
2021 	req = nfs_lock_and_join_requests(page);
2022 
2023 	if (IS_ERR(req)) {
2024 		ret = PTR_ERR(req);
2025 	} else if (req) {
2026 		/* all requests from this page have been cancelled by
2027 		 * nfs_lock_and_join_requests, so just remove the head
2028 		 * request from the inode / page_private pointer and
2029 		 * release it */
2030 		nfs_inode_remove_request(req);
2031 		nfs_unlock_and_release_request(req);
2032 	}
2033 
2034 	return ret;
2035 }
2036 
2037 /*
2038  * Write back all requests on one page - we do this before reading it.
2039  */
nfs_wb_page(struct inode * inode,struct page * page)2040 int nfs_wb_page(struct inode *inode, struct page *page)
2041 {
2042 	loff_t range_start = page_file_offset(page);
2043 	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2044 	struct writeback_control wbc = {
2045 		.sync_mode = WB_SYNC_ALL,
2046 		.nr_to_write = 0,
2047 		.range_start = range_start,
2048 		.range_end = range_end,
2049 	};
2050 	int ret;
2051 
2052 	trace_nfs_writeback_page_enter(inode);
2053 
2054 	for (;;) {
2055 		wait_on_page_writeback(page);
2056 		if (clear_page_dirty_for_io(page)) {
2057 			ret = nfs_writepage_locked(page, &wbc);
2058 			if (ret < 0)
2059 				goto out_error;
2060 			continue;
2061 		}
2062 		ret = 0;
2063 		if (!PagePrivate(page))
2064 			break;
2065 		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2066 		if (ret < 0)
2067 			goto out_error;
2068 	}
2069 out_error:
2070 	trace_nfs_writeback_page_exit(inode, ret);
2071 	return ret;
2072 }
2073 
2074 #ifdef CONFIG_MIGRATION
nfs_migrate_page(struct address_space * mapping,struct page * newpage,struct page * page,enum migrate_mode mode)2075 int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2076 		struct page *page, enum migrate_mode mode)
2077 {
2078 	/*
2079 	 * If PagePrivate is set, then the page is currently associated with
2080 	 * an in-progress read or write request. Don't try to migrate it.
2081 	 *
2082 	 * FIXME: we could do this in principle, but we'll need a way to ensure
2083 	 *        that we can safely release the inode reference while holding
2084 	 *        the page lock.
2085 	 */
2086 	if (PagePrivate(page))
2087 		return -EBUSY;
2088 
2089 	if (!nfs_fscache_release_page(page, GFP_KERNEL))
2090 		return -EBUSY;
2091 
2092 	return migrate_page(mapping, newpage, page, mode);
2093 }
2094 #endif
2095 
nfs_init_writepagecache(void)2096 int __init nfs_init_writepagecache(void)
2097 {
2098 	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2099 					     sizeof(struct nfs_pgio_header),
2100 					     0, SLAB_HWCACHE_ALIGN,
2101 					     NULL);
2102 	if (nfs_wdata_cachep == NULL)
2103 		return -ENOMEM;
2104 
2105 	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
2106 						     nfs_wdata_cachep);
2107 	if (nfs_wdata_mempool == NULL)
2108 		goto out_destroy_write_cache;
2109 
2110 	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
2111 					     sizeof(struct nfs_commit_data),
2112 					     0, SLAB_HWCACHE_ALIGN,
2113 					     NULL);
2114 	if (nfs_cdata_cachep == NULL)
2115 		goto out_destroy_write_mempool;
2116 
2117 	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2118 						      nfs_cdata_cachep);
2119 	if (nfs_commit_mempool == NULL)
2120 		goto out_destroy_commit_cache;
2121 
2122 	/*
2123 	 * NFS congestion size, scale with available memory.
2124 	 *
2125 	 *  64MB:    8192k
2126 	 * 128MB:   11585k
2127 	 * 256MB:   16384k
2128 	 * 512MB:   23170k
2129 	 *   1GB:   32768k
2130 	 *   2GB:   46340k
2131 	 *   4GB:   65536k
2132 	 *   8GB:   92681k
2133 	 *  16GB:  131072k
2134 	 *
2135 	 * This allows larger machines to have larger/more transfers.
2136 	 * Limit the default to 256M
2137 	 */
2138 	nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10);
2139 	if (nfs_congestion_kb > 256*1024)
2140 		nfs_congestion_kb = 256*1024;
2141 
2142 	return 0;
2143 
2144 out_destroy_commit_cache:
2145 	kmem_cache_destroy(nfs_cdata_cachep);
2146 out_destroy_write_mempool:
2147 	mempool_destroy(nfs_wdata_mempool);
2148 out_destroy_write_cache:
2149 	kmem_cache_destroy(nfs_wdata_cachep);
2150 	return -ENOMEM;
2151 }
2152 
nfs_destroy_writepagecache(void)2153 void nfs_destroy_writepagecache(void)
2154 {
2155 	mempool_destroy(nfs_commit_mempool);
2156 	kmem_cache_destroy(nfs_cdata_cachep);
2157 	mempool_destroy(nfs_wdata_mempool);
2158 	kmem_cache_destroy(nfs_wdata_cachep);
2159 }
2160 
2161 static const struct nfs_rw_ops nfs_rw_write_ops = {
2162 	.rw_alloc_header	= nfs_writehdr_alloc,
2163 	.rw_free_header		= nfs_writehdr_free,
2164 	.rw_done		= nfs_writeback_done,
2165 	.rw_result		= nfs_writeback_result,
2166 	.rw_initiate		= nfs_initiate_write,
2167 };
2168