Lines Matching refs:sdio
158 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
160 return sdio->tail - sdio->head; in dio_pages_present()
166 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
170 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
171 &sdio->from); in dio_refill_pages()
173 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
184 sdio->head = 0; in dio_refill_pages()
185 sdio->tail = 1; in dio_refill_pages()
186 sdio->from = 0; in dio_refill_pages()
187 sdio->to = PAGE_SIZE; in dio_refill_pages()
192 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
193 ret += sdio->from; in dio_refill_pages()
194 sdio->head = 0; in dio_refill_pages()
195 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
196 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
209 struct dio_submit *sdio) in dio_get_page() argument
211 if (dio_pages_present(sdio) == 0) { in dio_get_page()
214 ret = dio_refill_pages(dio, sdio); in dio_get_page()
217 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
219 return dio->pages[sdio->head]; in dio_get_page()
428 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
450 sdio->bio = bio; in dio_bio_alloc()
451 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
461 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
463 struct bio *bio = sdio->bio; in dio_bio_submit()
477 if (sdio->submit_io) { in dio_bio_submit()
478 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
483 sdio->bio = NULL; in dio_bio_submit()
484 sdio->boundary = 0; in dio_bio_submit()
485 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
491 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
493 while (sdio->head < sdio->tail) in dio_cleanup()
494 put_page(dio->pages[sdio->head++]); in dio_cleanup()
591 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
595 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
609 sdio->reap_counter = 0; in dio_bio_reap()
673 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
681 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
690 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
691 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
692 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
693 sdio->blkfactor; in get_more_blocks()
717 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
732 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
738 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
741 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
742 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
744 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
745 sdio->boundary = 0; in dio_new_bio()
757 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
761 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
762 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
763 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
767 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
768 sdio->pages_in_io--; in dio_bio_add_page()
769 get_page(sdio->cur_page); in dio_bio_add_page()
770 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
771 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
789 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
794 if (sdio->bio) { in dio_send_cur_page()
795 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
796 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
797 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
813 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
815 dio_bio_submit(dio, sdio); in dio_send_cur_page()
818 if (sdio->bio == NULL) { in dio_send_cur_page()
819 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
824 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
825 dio_bio_submit(dio, sdio); in dio_send_cur_page()
826 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
828 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
854 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
859 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ in submit_page_section()
871 if (sdio->cur_page == page && in submit_page_section()
872 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
873 sdio->cur_page_block + in submit_page_section()
874 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
875 sdio->cur_page_len += len; in submit_page_section()
882 if (sdio->cur_page) { in submit_page_section()
883 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
884 put_page(sdio->cur_page); in submit_page_section()
885 sdio->cur_page = NULL; in submit_page_section()
891 sdio->cur_page = page; in submit_page_section()
892 sdio->cur_page_offset = offset; in submit_page_section()
893 sdio->cur_page_len = len; in submit_page_section()
894 sdio->cur_page_block = blocknr; in submit_page_section()
895 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
902 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
903 if (sdio->bio) in submit_page_section()
904 dio_bio_submit(dio, sdio); in submit_page_section()
905 put_page(sdio->cur_page); in submit_page_section()
906 sdio->cur_page = NULL; in submit_page_section()
920 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
928 sdio->start_zero_done = 1; in dio_zero_block()
929 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
932 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
933 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
945 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
948 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
949 sdio->next_block_for_io, map_bh)) in dio_zero_block()
952 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
971 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
974 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
975 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
978 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
982 page = dio_get_page(dio, sdio); in do_direct_IO()
987 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
988 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
989 sdio->head++; in do_direct_IO()
996 if (sdio->blocks_available == 0) { in do_direct_IO()
1003 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
1011 sdio->blocks_available = in do_direct_IO()
1013 sdio->next_block_for_io = in do_direct_IO()
1014 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
1022 if (!sdio->blkfactor) in do_direct_IO()
1025 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
1026 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
1040 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
1041 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1060 if (sdio->block_in_file >= in do_direct_IO()
1067 sdio->block_in_file++; in do_direct_IO()
1078 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1079 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1085 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1089 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1095 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1096 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1097 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1100 sdio->next_block_for_io, in do_direct_IO()
1106 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1108 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1111 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1113 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1114 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1186 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1303 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1304 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1305 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1307 sdio.get_block = get_block; in do_blockdev_direct_IO()
1309 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1310 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1311 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1319 sdio.iter = iter; in do_blockdev_direct_IO()
1320 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1326 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1327 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1329 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1333 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1335 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1348 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1350 if (sdio.cur_page) { in do_blockdev_direct_IO()
1353 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1356 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1357 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1359 if (sdio.bio) in do_blockdev_direct_IO()
1360 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1368 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()