Lines Matching refs:addr
109 static void stage2_dissolve_pmd(struct kvm *kvm, phys_addr_t addr, pmd_t *pmd) in stage2_dissolve_pmd() argument
115 kvm_tlb_flush_vmid_ipa(kvm, addr); in stage2_dissolve_pmd()
151 static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr) in clear_stage2_pgd_entry() argument
155 kvm_tlb_flush_vmid_ipa(kvm, addr); in clear_stage2_pgd_entry()
160 static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr) in clear_stage2_pud_entry() argument
165 kvm_tlb_flush_vmid_ipa(kvm, addr); in clear_stage2_pud_entry()
170 static void clear_stage2_pmd_entry(struct kvm *kvm, pmd_t *pmd, phys_addr_t addr) in clear_stage2_pmd_entry() argument
175 kvm_tlb_flush_vmid_ipa(kvm, addr); in clear_stage2_pmd_entry()
234 phys_addr_t addr, phys_addr_t end) in unmap_stage2_ptes() argument
236 phys_addr_t start_addr = addr; in unmap_stage2_ptes()
239 start_pte = pte = pte_offset_kernel(pmd, addr); in unmap_stage2_ptes()
245 kvm_tlb_flush_vmid_ipa(kvm, addr); in unmap_stage2_ptes()
253 } while (pte++, addr += PAGE_SIZE, addr != end); in unmap_stage2_ptes()
260 phys_addr_t addr, phys_addr_t end) in unmap_stage2_pmds() argument
262 phys_addr_t next, start_addr = addr; in unmap_stage2_pmds()
265 start_pmd = pmd = stage2_pmd_offset(pud, addr); in unmap_stage2_pmds()
267 next = stage2_pmd_addr_end(addr, end); in unmap_stage2_pmds()
273 kvm_tlb_flush_vmid_ipa(kvm, addr); in unmap_stage2_pmds()
279 unmap_stage2_ptes(kvm, pmd, addr, next); in unmap_stage2_pmds()
282 } while (pmd++, addr = next, addr != end); in unmap_stage2_pmds()
289 phys_addr_t addr, phys_addr_t end) in unmap_stage2_puds() argument
291 phys_addr_t next, start_addr = addr; in unmap_stage2_puds()
294 start_pud = pud = stage2_pud_offset(pgd, addr); in unmap_stage2_puds()
296 next = stage2_pud_addr_end(addr, end); in unmap_stage2_puds()
302 kvm_tlb_flush_vmid_ipa(kvm, addr); in unmap_stage2_puds()
306 unmap_stage2_pmds(kvm, pud, addr, next); in unmap_stage2_puds()
309 } while (pud++, addr = next, addr != end); in unmap_stage2_puds()
330 phys_addr_t addr = start, end = start + size; in __unmap_stage2_range() local
336 pgd = kvm->arch.pgd + stage2_pgd_index(addr); in __unmap_stage2_range()
345 next = stage2_pgd_addr_end(addr, end); in __unmap_stage2_range()
347 unmap_stage2_puds(kvm, pgd, addr, next); in __unmap_stage2_range()
354 } while (pgd++, addr = next, addr != end); in __unmap_stage2_range()
363 phys_addr_t addr, phys_addr_t end) in stage2_flush_ptes() argument
367 pte = pte_offset_kernel(pmd, addr); in stage2_flush_ptes()
371 } while (pte++, addr += PAGE_SIZE, addr != end); in stage2_flush_ptes()
375 phys_addr_t addr, phys_addr_t end) in stage2_flush_pmds() argument
380 pmd = stage2_pmd_offset(pud, addr); in stage2_flush_pmds()
382 next = stage2_pmd_addr_end(addr, end); in stage2_flush_pmds()
387 stage2_flush_ptes(kvm, pmd, addr, next); in stage2_flush_pmds()
389 } while (pmd++, addr = next, addr != end); in stage2_flush_pmds()
393 phys_addr_t addr, phys_addr_t end) in stage2_flush_puds() argument
398 pud = stage2_pud_offset(pgd, addr); in stage2_flush_puds()
400 next = stage2_pud_addr_end(addr, end); in stage2_flush_puds()
405 stage2_flush_pmds(kvm, pud, addr, next); in stage2_flush_puds()
407 } while (pud++, addr = next, addr != end); in stage2_flush_puds()
413 phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; in stage2_flush_memslot() local
414 phys_addr_t end = addr + PAGE_SIZE * memslot->npages; in stage2_flush_memslot()
418 pgd = kvm->arch.pgd + stage2_pgd_index(addr); in stage2_flush_memslot()
420 next = stage2_pgd_addr_end(addr, end); in stage2_flush_memslot()
422 stage2_flush_puds(kvm, pgd, addr, next); in stage2_flush_memslot()
423 } while (pgd++, addr = next, addr != end); in stage2_flush_memslot()
476 static void unmap_hyp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) in unmap_hyp_ptes() argument
480 start_pte = pte = pte_offset_kernel(pmd, addr); in unmap_hyp_ptes()
486 } while (pte++, addr += PAGE_SIZE, addr != end); in unmap_hyp_ptes()
492 static void unmap_hyp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) in unmap_hyp_pmds() argument
497 start_pmd = pmd = pmd_offset(pud, addr); in unmap_hyp_pmds()
499 next = pmd_addr_end(addr, end); in unmap_hyp_pmds()
502 unmap_hyp_ptes(pmd, addr, next); in unmap_hyp_pmds()
503 } while (pmd++, addr = next, addr != end); in unmap_hyp_pmds()
509 static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end) in unmap_hyp_puds() argument
514 start_pud = pud = pud_offset(pgd, addr); in unmap_hyp_puds()
516 next = pud_addr_end(addr, end); in unmap_hyp_puds()
519 unmap_hyp_pmds(pud, addr, next); in unmap_hyp_puds()
520 } while (pud++, addr = next, addr != end); in unmap_hyp_puds()
526 static unsigned int kvm_pgd_index(unsigned long addr, unsigned int ptrs_per_pgd) in kvm_pgd_index() argument
528 return (addr >> PGDIR_SHIFT) & (ptrs_per_pgd - 1); in kvm_pgd_index()
535 phys_addr_t addr = start, end = start + size; in __unmap_hyp_range() local
542 pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd); in __unmap_hyp_range()
544 next = pgd_addr_end(addr, end); in __unmap_hyp_range()
546 unmap_hyp_puds(pgd, addr, next); in __unmap_hyp_range()
547 } while (pgd++, addr = next, addr != end); in __unmap_hyp_range()
612 unsigned long addr; in create_hyp_pte_mappings() local
614 addr = start; in create_hyp_pte_mappings()
616 pte = pte_offset_kernel(pmd, addr); in create_hyp_pte_mappings()
620 } while (addr += PAGE_SIZE, addr != end); in create_hyp_pte_mappings()
629 unsigned long addr, next; in create_hyp_pmd_mappings() local
631 addr = start; in create_hyp_pmd_mappings()
633 pmd = pmd_offset(pud, addr); in create_hyp_pmd_mappings()
638 pte = pte_alloc_one_kernel(NULL, addr); in create_hyp_pmd_mappings()
647 next = pmd_addr_end(addr, end); in create_hyp_pmd_mappings()
649 create_hyp_pte_mappings(pmd, addr, next, pfn, prot); in create_hyp_pmd_mappings()
650 pfn += (next - addr) >> PAGE_SHIFT; in create_hyp_pmd_mappings()
651 } while (addr = next, addr != end); in create_hyp_pmd_mappings()
662 unsigned long addr, next; in create_hyp_pud_mappings() local
665 addr = start; in create_hyp_pud_mappings()
667 pud = pud_offset(pgd, addr); in create_hyp_pud_mappings()
670 pmd = pmd_alloc_one(NULL, addr); in create_hyp_pud_mappings()
679 next = pud_addr_end(addr, end); in create_hyp_pud_mappings()
680 ret = create_hyp_pmd_mappings(pud, addr, next, pfn, prot); in create_hyp_pud_mappings()
683 pfn += (next - addr) >> PAGE_SHIFT; in create_hyp_pud_mappings()
684 } while (addr = next, addr != end); in create_hyp_pud_mappings()
695 unsigned long addr, next; in __create_hyp_mappings() local
699 addr = start & PAGE_MASK; in __create_hyp_mappings()
702 pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd); in __create_hyp_mappings()
705 pud = pud_alloc_one(NULL, addr); in __create_hyp_mappings()
715 next = pgd_addr_end(addr, end); in __create_hyp_mappings()
716 err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot); in __create_hyp_mappings()
719 pfn += (next - addr) >> PAGE_SHIFT; in __create_hyp_mappings()
720 } while (addr = next, addr != end); in __create_hyp_mappings()
836 unsigned long addr; in create_hyp_io_mappings() local
849 &addr, PAGE_HYP_DEVICE); in create_hyp_io_mappings()
857 *haddr = (void __iomem *)addr; in create_hyp_io_mappings()
870 unsigned long addr; in create_hyp_exec_mappings() local
876 &addr, PAGE_HYP_EXEC); in create_hyp_exec_mappings()
882 *haddr = (void *)addr; in create_hyp_exec_mappings()
919 phys_addr_t addr = memslot->base_gfn << PAGE_SHIFT; in stage2_unmap_memslot() local
949 gpa_t gpa = addr + (vm_start - memslot->userspace_addr); in stage2_unmap_memslot()
1008 phys_addr_t addr) in stage2_get_pud() argument
1013 pgd = kvm->arch.pgd + stage2_pgd_index(addr); in stage2_get_pud()
1022 return stage2_pud_offset(pgd, addr); in stage2_get_pud()
1026 phys_addr_t addr) in stage2_get_pmd() argument
1031 pud = stage2_get_pud(kvm, cache, addr); in stage2_get_pmd()
1043 return stage2_pmd_offset(pud, addr); in stage2_get_pmd()
1047 *cache, phys_addr_t addr, const pmd_t *new_pmd) in stage2_set_pmd_huge() argument
1051 pmd = stage2_get_pmd(kvm, cache, addr); in stage2_set_pmd_huge()
1084 kvm_tlb_flush_vmid_ipa(kvm, addr); in stage2_set_pmd_huge()
1093 static bool stage2_is_exec(struct kvm *kvm, phys_addr_t addr) in stage2_is_exec() argument
1098 pmdp = stage2_get_pmd(kvm, NULL, addr); in stage2_is_exec()
1105 ptep = pte_offset_kernel(pmdp, addr); in stage2_is_exec()
1113 phys_addr_t addr, const pte_t *new_pte, in stage2_set_pte() argument
1124 pmd = stage2_get_pmd(kvm, cache, addr); in stage2_set_pte()
1138 stage2_dissolve_pmd(kvm, addr, pmd); in stage2_set_pte()
1149 pte = pte_offset_kernel(pmd, addr); in stage2_set_pte()
1162 kvm_tlb_flush_vmid_ipa(kvm, addr); in stage2_set_pte()
1203 phys_addr_t addr, end; in kvm_phys_addr_ioremap() local
1211 for (addr = guest_ipa; addr < end; addr += PAGE_SIZE) { in kvm_phys_addr_ioremap()
1222 ret = stage2_set_pte(kvm, &cache, addr, &pte, in kvm_phys_addr_ioremap()
1300 static void stage2_wp_ptes(pmd_t *pmd, phys_addr_t addr, phys_addr_t end) in stage2_wp_ptes() argument
1304 pte = pte_offset_kernel(pmd, addr); in stage2_wp_ptes()
1310 } while (pte++, addr += PAGE_SIZE, addr != end); in stage2_wp_ptes()
1319 static void stage2_wp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end) in stage2_wp_pmds() argument
1324 pmd = stage2_pmd_offset(pud, addr); in stage2_wp_pmds()
1327 next = stage2_pmd_addr_end(addr, end); in stage2_wp_pmds()
1333 stage2_wp_ptes(pmd, addr, next); in stage2_wp_pmds()
1336 } while (pmd++, addr = next, addr != end); in stage2_wp_pmds()
1347 static void stage2_wp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end) in stage2_wp_puds() argument
1352 pud = stage2_pud_offset(pgd, addr); in stage2_wp_puds()
1354 next = stage2_pud_addr_end(addr, end); in stage2_wp_puds()
1358 stage2_wp_pmds(pud, addr, next); in stage2_wp_puds()
1360 } while (pud++, addr = next, addr != end); in stage2_wp_puds()
1369 static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end) in stage2_wp_range() argument
1374 pgd = kvm->arch.pgd + stage2_pgd_index(addr); in stage2_wp_range()
1388 next = stage2_pgd_addr_end(addr, end); in stage2_wp_range()
1390 stage2_wp_puds(pgd, addr, next); in stage2_wp_range()
1391 } while (pgd++, addr = next, addr != end); in stage2_wp_range()