151 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
159 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
160 		bytes = i->count;  in copy_page_to_iter_iovec()
162 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
166 	wanted = bytes;  in copy_page_to_iter_iovec()
170 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
181 		bytes -= copy;  in copy_page_to_iter_iovec()
183 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
186 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
191 			bytes -= copy;  in copy_page_to_iter_iovec()
193 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
200 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
210 	bytes -= copy;  in copy_page_to_iter_iovec()
211 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
214 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
219 		bytes -= copy;  in copy_page_to_iter_iovec()
228 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
232 	return wanted - bytes;  in copy_page_to_iter_iovec()
235 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
243 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
244 		bytes = i->count;  in copy_page_from_iter_iovec()
246 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
250 	wanted = bytes;  in copy_page_from_iter_iovec()
254 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
265 		bytes -= copy;  in copy_page_from_iter_iovec()
267 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
270 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
275 			bytes -= copy;  in copy_page_from_iter_iovec()
277 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
284 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
294 	bytes -= copy;  in copy_page_from_iter_iovec()
295 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
298 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
303 		bytes -= copy;  in copy_page_from_iter_iovec()
312 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
316 	return wanted - bytes;  in copy_page_from_iter_iovec()
362 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
370 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
371 		bytes = i->count;  in copy_page_to_iter_pipe()
373 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
385 			buf->len += bytes;  in copy_page_to_iter_pipe()
386 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
399 	buf->len = bytes;  in copy_page_to_iter_pipe()
400 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
403 	i->count -= bytes;  in copy_page_to_iter_pipe()
404 	return bytes;  in copy_page_to_iter_pipe()
414 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
422 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
535 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
545 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter()
556 	i->count -= bytes;  in copy_pipe_to_iter()
557 	return bytes;  in copy_pipe_to_iter()
560 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
564 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
567 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
574 	return bytes;  in _copy_to_iter()
601 static size_t copy_pipe_to_iter_mcsafe(const void *addr, size_t bytes,  in copy_pipe_to_iter_mcsafe()  argument
611 	bytes = n = push_pipe(i, bytes, &idx, &off);  in copy_pipe_to_iter_mcsafe()
658 size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter_mcsafe()  argument
664 		return copy_pipe_to_iter_mcsafe(addr, bytes, i);  in _copy_to_iter_mcsafe()
667 	iterate_and_advance(i, bytes, v,  in _copy_to_iter_mcsafe()
674 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
675 			return bytes;  in _copy_to_iter_mcsafe()
683 			bytes = curr_addr - s_addr - rem;  in _copy_to_iter_mcsafe()
684 			return bytes;  in _copy_to_iter_mcsafe()
689 	return bytes;  in _copy_to_iter_mcsafe()
694 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
703 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
710 	return bytes;  in _copy_from_iter()
714 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
721 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
726 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
736 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
741 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
748 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
756 	return bytes;  in _copy_from_iter_nocache()
775 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
782 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
791 	return bytes;  in _copy_from_iter_flushcache()
796 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
803 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
805 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
815 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
844 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
847 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
851 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
855 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
857 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
861 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
864 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
872 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
876 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
880 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
889 	bytes = n = push_pipe(i, bytes, &idx, &off);  in pipe_zero()
900 	i->count -= bytes;  in pipe_zero()
901 	return bytes;  in pipe_zero()
904 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
907 		return pipe_zero(bytes, i);  in iov_iter_zero()
908 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
914 	return bytes;  in iov_iter_zero()
919 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
922 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
931 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
938 	return bytes;  in iov_iter_copy_from_user_atomic()
1324 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1335 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1362 	return bytes;  in csum_and_copy_from_iter()
1366 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1377 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1379 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1406 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1411 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_to_iter()  argument
1422 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1449 	return bytes;  in csum_and_copy_to_iter()
1595 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,  in iov_iter_for_each_range()  argument
1601 	if (!bytes)  in iov_iter_for_each_range()
1604 	iterate_all_kinds(i, bytes, v, -EINVAL, ({  in iov_iter_for_each_range()