809 	size_t loop_len;  in scif_rma_local_cpu_copy()  local
814 	loop_len = PAGE_SIZE - offset_in_page;  in scif_rma_local_cpu_copy()
816 	if (rem_len < loop_len)  in scif_rma_local_cpu_copy()
817 		loop_len = rem_len;  in scif_rma_local_cpu_copy()
819 	window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
823 		memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
825 		memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
827 	offset += loop_len;  in scif_rma_local_cpu_copy()
828 	temp += loop_len;  in scif_rma_local_cpu_copy()
829 	rem_len -= loop_len;  in scif_rma_local_cpu_copy()
839 		loop_len = min(PAGE_SIZE, rem_len);  in scif_rma_local_cpu_copy()
840 		window_virt = _get_local_va(offset, window, loop_len);  in scif_rma_local_cpu_copy()
844 			memcpy(temp, window_virt, loop_len);  in scif_rma_local_cpu_copy()
846 			memcpy(window_virt, temp, loop_len);  in scif_rma_local_cpu_copy()
847 		offset	+= loop_len;  in scif_rma_local_cpu_copy()
848 		temp	+= loop_len;  in scif_rma_local_cpu_copy()
849 		rem_len	-= loop_len;  in scif_rma_local_cpu_copy()
888 	size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;  in scif_rma_list_dma_copy_unaligned()  local
908 		loop_len = L1_CACHE_BYTES - offset_in_ca;  in scif_rma_list_dma_copy_unaligned()
909 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_unaligned()
911 						  loop_len,  in scif_rma_list_dma_copy_unaligned()
918 						loop_len,  in scif_rma_list_dma_copy_unaligned()
920 						!(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
923 						  loop_len, work->ordered &&  in scif_rma_list_dma_copy_unaligned()
924 						  !(remaining_len - loop_len));  in scif_rma_list_dma_copy_unaligned()
925 		iounmap_remote(window_virt_addr, loop_len, work);  in scif_rma_list_dma_copy_unaligned()
927 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
928 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
929 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
930 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
953 		loop_len = min(nr_contig_bytes, remaining_len);  in scif_rma_list_dma_copy_unaligned()
956 			    !(remaining_len - loop_len) &&  in scif_rma_list_dma_copy_unaligned()
957 			    loop_len != L1_CACHE_BYTES) {  in scif_rma_list_dma_copy_unaligned()
972 							    loop_len -  in scif_rma_list_dma_copy_unaligned()
985 				offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
986 				temp_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
987 				window_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
988 				remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_unaligned()
989 				loop_len = remaining_len;  in scif_rma_list_dma_copy_unaligned()
996 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1012 							    loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1026 					window_dma_addr, loop_len, 0);  in scif_rma_list_dma_copy_unaligned()
1040 		offset += loop_len;  in scif_rma_list_dma_copy_unaligned()
1041 		temp += loop_len;  in scif_rma_list_dma_copy_unaligned()
1042 		temp_phys += loop_len;  in scif_rma_list_dma_copy_unaligned()
1043 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_unaligned()
1108 	size_t loop_len, remaining_len, src_contig_bytes = 0;  in _scif_rma_list_dma_copy_aligned()  local
1150 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in _scif_rma_list_dma_copy_aligned()
1151 		loop_len = min(loop_len, remaining_len);  in _scif_rma_list_dma_copy_aligned()
1152 		if (work->ordered && !(remaining_len - loop_len)) {  in _scif_rma_list_dma_copy_aligned()
1161 							 loop_len - 1,  in _scif_rma_list_dma_copy_aligned()
1172 			src_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1173 			dst_offset += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1174 			src_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1175 			dst_dma_addr += (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1176 			remaining_len -= (loop_len - 1);  in _scif_rma_list_dma_copy_aligned()
1177 			loop_len = remaining_len;  in _scif_rma_list_dma_copy_aligned()
1181 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1194 					src_dma_addr, loop_len, 0);  in _scif_rma_list_dma_copy_aligned()
1205 		src_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1206 		dst_offset += loop_len;  in _scif_rma_list_dma_copy_aligned()
1207 		remaining_len -= loop_len;  in _scif_rma_list_dma_copy_aligned()
1226 	size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;  in scif_rma_list_dma_copy_aligned()  local
1248 		loop_len = L1_CACHE_BYTES - src_cache_off;  in scif_rma_list_dma_copy_aligned()
1249 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1254 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1257 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1263 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1266 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1270 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1274 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1275 						remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1278 			scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1279 						  remaining_len == loop_len ?  in scif_rma_list_dma_copy_aligned()
1282 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1284 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1285 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1286 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1287 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1317 		loop_len = min(src_contig_bytes, dst_contig_bytes);  in scif_rma_list_dma_copy_aligned()
1318 		loop_len = min(loop_len, remaining_len);  in scif_rma_list_dma_copy_aligned()
1320 		    !(remaining_len - loop_len)) {  in scif_rma_list_dma_copy_aligned()
1332 							 loop_len -  in scif_rma_list_dma_copy_aligned()
1345 			src_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1346 			dst_offset += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1347 			src_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1348 			dst_dma_addr += (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1349 			remaining_len -= (loop_len - L1_CACHE_BYTES);  in scif_rma_list_dma_copy_aligned()
1350 			loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1355 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1369 							 loop_len, 0);  in scif_rma_list_dma_copy_aligned()
1381 		src_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1382 		dst_offset += loop_len;  in scif_rma_list_dma_copy_aligned()
1383 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1387 		loop_len = remaining_len;  in scif_rma_list_dma_copy_aligned()
1409 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1412 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1419 						 loop_len);  in scif_rma_list_dma_copy_aligned()
1422 						  loop_len,  in scif_rma_list_dma_copy_aligned()
1426 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1431 			scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,  in scif_rma_list_dma_copy_aligned()
1435 						  loop_len, work->ordered);  in scif_rma_list_dma_copy_aligned()
1437 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1440 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_dma_copy_aligned()
1441 		remaining_len -= loop_len;  in scif_rma_list_dma_copy_aligned()
1459 	size_t loop_len, remaining_len;  in scif_rma_list_cpu_copy()  local
1476 		loop_len = min(PAGE_SIZE -  in scif_rma_list_cpu_copy()
1482 						 loop_len);  in scif_rma_list_cpu_copy()
1485 						  loop_len,  in scif_rma_list_cpu_copy()
1495 						 loop_len);  in scif_rma_list_cpu_copy()
1498 						  loop_len,  in scif_rma_list_cpu_copy()
1503 				iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1509 			memcpy(dst_virt, src_virt, loop_len);  in scif_rma_list_cpu_copy()
1513 					    src_virt, loop_len);  in scif_rma_list_cpu_copy()
1517 					      loop_len);  in scif_rma_list_cpu_copy()
1520 			iounmap_remote(src_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1523 			iounmap_remote(dst_virt, loop_len, work);  in scif_rma_list_cpu_copy()
1525 		src_offset += loop_len;  in scif_rma_list_cpu_copy()
1526 		dst_offset += loop_len;  in scif_rma_list_cpu_copy()
1527 		remaining_len -= loop_len;  in scif_rma_list_cpu_copy()