| Message ID | 20200626131000.5207-4-bharata@linux.ibm.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | Off-load TLB invalidations to host for !GTSE | expand |
Bharata B Rao <bharata@linux.ibm.com> writes: > From: Nicholas Piggin <npiggin@gmail.com> > > When platform doesn't support GTSE, let TLB invalidation requests > for radix guests be off-loaded to the host using H_RPT_INVALIDATE > hcall. > Reviewed-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com> > Signed-off-by: Nicholas Piggin <npiggin@gmail.com> > Signed-off-by: Bharata B Rao <bharata@linux.ibm.com> > [hcall wrapper, error path handling and renames] > --- > .../include/asm/book3s/64/tlbflush-radix.h | 15 ++++ > arch/powerpc/include/asm/hvcall.h | 34 +++++++- > arch/powerpc/include/asm/plpar_wrappers.h | 50 +++++++++++ > arch/powerpc/mm/book3s64/radix_tlb.c | 82 +++++++++++++++++-- > 4 files changed, 173 insertions(+), 8 deletions(-) > > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > index ca8db193ae38..e7cf50358411 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > @@ -2,10 +2,25 @@ > #ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H > #define _ASM_POWERPC_TLBFLUSH_RADIX_H > > +#include <asm/hvcall.h> > + > struct vm_area_struct; > struct mm_struct; > struct mmu_gather; > > +static inline u64 psize_to_h_rpti(unsigned long psize) > +{ > + if (psize == MMU_PAGE_4K) > + return H_RPTI_PAGE_4K; > + if (psize == MMU_PAGE_64K) > + return H_RPTI_PAGE_64K; > + if (psize == MMU_PAGE_2M) > + return H_RPTI_PAGE_2M; > + if (psize == MMU_PAGE_1G) > + return H_RPTI_PAGE_1G; > + return H_RPTI_PAGE_ALL; > +} > + > static inline int mmu_get_ap(int psize) > { > return mmu_psize_defs[psize].ap; > diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h > index e90c073e437e..43486e773bd6 100644 > --- a/arch/powerpc/include/asm/hvcall.h > +++ b/arch/powerpc/include/asm/hvcall.h > @@ -305,7 +305,8 @@ > #define H_SCM_UNBIND_ALL 0x3FC > #define H_SCM_HEALTH 0x400 > #define H_SCM_PERFORMANCE_STATS 0x418 > -#define MAX_HCALL_OPCODE H_SCM_PERFORMANCE_STATS > +#define H_RPT_INVALIDATE 0x448 > +#define MAX_HCALL_OPCODE H_RPT_INVALIDATE > > /* Scope args for H_SCM_UNBIND_ALL */ > #define H_UNBIND_SCOPE_ALL (0x1) > @@ -389,6 +390,37 @@ > #define PROC_TABLE_RADIX 0x04 > #define PROC_TABLE_GTSE 0x01 > > +/* > + * Defines for > + * H_RPT_INVALIDATE - Invalidate RPT translation lookaside information. > + */ > + > +/* Type of translation to invalidate (type) */ > +#define H_RPTI_TYPE_NESTED 0x0001 /* Invalidate nested guest partition-scope */ > +#define H_RPTI_TYPE_TLB 0x0002 /* Invalidate TLB */ > +#define H_RPTI_TYPE_PWC 0x0004 /* Invalidate Page Walk Cache */ > +/* Invalidate Process Table Entries if H_RPTI_TYPE_NESTED is clear */ > +#define H_RPTI_TYPE_PRT 0x0008 > +/* Invalidate Partition Table Entries if H_RPTI_TYPE_NESTED is set */ > +#define H_RPTI_TYPE_PAT 0x0008 > +#define H_RPTI_TYPE_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ > + H_RPTI_TYPE_PRT) > +#define H_RPTI_TYPE_NESTED_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ > + H_RPTI_TYPE_PAT) > + > +/* Invalidation targets (target) */ > +#define H_RPTI_TARGET_CMMU 0x01 /* All virtual processors in the partition */ > +#define H_RPTI_TARGET_CMMU_LOCAL 0x02 /* Current virtual processor */ > +/* All nest/accelerator agents in use by the partition */ > +#define H_RPTI_TARGET_NMMU 0x04 > + > +/* Page size mask (page sizes) */ > +#define H_RPTI_PAGE_4K 0x01 > +#define H_RPTI_PAGE_64K 0x02 > +#define H_RPTI_PAGE_2M 0x04 > +#define H_RPTI_PAGE_1G 0x08 > +#define H_RPTI_PAGE_ALL (-1UL) > + > #ifndef __ASSEMBLY__ > #include <linux/types.h> > > diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h > index 4497c8afb573..a184923abd07 100644 > --- a/arch/powerpc/include/asm/plpar_wrappers.h > +++ b/arch/powerpc/include/asm/plpar_wrappers.h > @@ -334,6 +334,49 @@ static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) > return rc; > } > > +/* > + * Wrapper to H_RPT_INVALIDATE hcall that handles return values appropriately > + * > + * - Returns H_SUCCESS on success > + * - For H_BUSY return value, we retry the hcall. > + * - For any other hcall failures, attempt a full flush once before > + * resorting to BUG(). > + * > + * Note: This hcall is expected to fail only very rarely. The correct > + * error recovery of killing the process/guest will be eventually > + * needed. > + */ > +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, > + u64 page_sizes, u64 start, u64 end) > +{ > + long rc; > + unsigned long all; > + > + while (true) { > + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, type, > + page_sizes, start, end); > + if (rc == H_BUSY) { > + cpu_relax(); > + continue; > + } else if (rc == H_SUCCESS) > + return rc; > + > + /* Flush request failed, try with a full flush once */ > + all = (type & H_RPTI_TYPE_NESTED) ? H_RPTI_TYPE_NESTED_ALL : > + H_RPTI_TYPE_ALL; > +retry: > + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, > + all, page_sizes, 0, -1UL); > + if (rc == H_BUSY) { > + cpu_relax(); > + goto retry; > + } else if (rc == H_SUCCESS) > + return rc; > + > + BUG(); > + } > +} > + > #else /* !CONFIG_PPC_PSERIES */ > > static inline long plpar_set_ciabr(unsigned long ciabr) > @@ -346,6 +389,13 @@ static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, > { > return 0; > } > + > +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, > + u64 page_sizes, u64 start, u64 end) > +{ > + return 0; > +} > + > #endif /* CONFIG_PPC_PSERIES */ > > #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ > diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c > index b5cc9b23cf02..180d8ddcf6e3 100644 > --- a/arch/powerpc/mm/book3s64/radix_tlb.c > +++ b/arch/powerpc/mm/book3s64/radix_tlb.c > @@ -16,6 +16,7 @@ > #include <asm/tlbflush.h> > #include <asm/trace.h> > #include <asm/cputhreads.h> > +#include <asm/plpar_wrappers.h> > > #define RIC_FLUSH_TLB 0 > #define RIC_FLUSH_PWC 1 > @@ -694,7 +695,14 @@ void radix__flush_tlb_mm(struct mm_struct *mm) > goto local; > } > > - if (cputlb_use_tlbie()) { > + if (!mmu_has_feature(MMU_FTR_GTSE)) { > + unsigned long tgt = H_RPTI_TARGET_CMMU; > + > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, > + H_RPTI_PAGE_ALL, 0, -1UL); > + } else if (cputlb_use_tlbie()) { > if (mm_needs_flush_escalation(mm)) > _tlbie_pid(pid, RIC_FLUSH_ALL); > else > @@ -727,7 +735,16 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) > goto local; > } > } > - if (cputlb_use_tlbie()) > + if (!mmu_has_feature(MMU_FTR_GTSE)) { > + unsigned long tgt = H_RPTI_TARGET_CMMU; > + unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | > + H_RPTI_TYPE_PRT; > + > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, type, > + H_RPTI_PAGE_ALL, 0, -1UL); > + } else if (cputlb_use_tlbie()) > _tlbie_pid(pid, RIC_FLUSH_ALL); > else > _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); > @@ -760,7 +777,19 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, > exit_flush_lazy_tlbs(mm); > goto local; > } > - if (cputlb_use_tlbie()) > + if (!mmu_has_feature(MMU_FTR_GTSE)) { > + unsigned long tgt, page_sizes, size; > + > + tgt = H_RPTI_TARGET_CMMU; > + page_sizes = psize_to_h_rpti(psize); > + size = 1UL << mmu_psize_to_shift(psize); > + > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, > + page_sizes, vmaddr, > + vmaddr + size); > + } else if (cputlb_use_tlbie()) > _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); > else > _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); > @@ -810,7 +839,14 @@ static inline void _tlbiel_kernel_broadcast(void) > */ > void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) > { > - if (cputlb_use_tlbie()) > + if (!mmu_has_feature(MMU_FTR_GTSE)) { > + unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU; > + unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | > + H_RPTI_TYPE_PRT; > + > + pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL, > + start, end); > + } else if (cputlb_use_tlbie()) > _tlbie_pid(0, RIC_FLUSH_ALL); > else > _tlbiel_kernel_broadcast(); > @@ -864,7 +900,17 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, > nr_pages > tlb_local_single_page_flush_ceiling); > } > > - if (full) { > + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { > + unsigned long tgt = H_RPTI_TARGET_CMMU; > + unsigned long page_sizes = psize_to_h_rpti(mmu_virtual_psize); > + > + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) > + page_sizes |= psize_to_h_rpti(MMU_PAGE_2M); > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, page_sizes, > + start, end); > + } else if (full) { > if (local) { > _tlbiel_pid(pid, RIC_FLUSH_TLB); > } else { > @@ -1046,7 +1092,17 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, > nr_pages > tlb_local_single_page_flush_ceiling); > } > > - if (full) { > + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { > + unsigned long tgt = H_RPTI_TARGET_CMMU; > + unsigned long type = H_RPTI_TYPE_TLB; > + unsigned long page_sizes = psize_to_h_rpti(psize); > + > + if (also_pwc) > + type |= H_RPTI_TYPE_PWC; > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, type, page_sizes, start, end); > + } else if (full) { > if (local) { > _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); > } else { > @@ -1111,7 +1167,19 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) > exit_flush_lazy_tlbs(mm); > goto local; > } > - if (cputlb_use_tlbie()) > + if (!mmu_has_feature(MMU_FTR_GTSE)) { > + unsigned long tgt, type, page_sizes; > + > + tgt = H_RPTI_TARGET_CMMU; > + type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | > + H_RPTI_TYPE_PRT; > + page_sizes = psize_to_h_rpti(mmu_virtual_psize); > + > + if (atomic_read(&mm->context.copros) > 0) > + tgt |= H_RPTI_TARGET_NMMU; > + pseries_rpt_invalidate(pid, tgt, type, page_sizes, > + addr, end); > + } else if (cputlb_use_tlbie()) > _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); > else > _tlbiel_va_range_multicast(mm, > -- > 2.21.3
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index ca8db193ae38..e7cf50358411 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -2,10 +2,25 @@ #ifndef _ASM_POWERPC_TLBFLUSH_RADIX_H #define _ASM_POWERPC_TLBFLUSH_RADIX_H +#include <asm/hvcall.h> + struct vm_area_struct; struct mm_struct; struct mmu_gather; +static inline u64 psize_to_h_rpti(unsigned long psize) +{ + if (psize == MMU_PAGE_4K) + return H_RPTI_PAGE_4K; + if (psize == MMU_PAGE_64K) + return H_RPTI_PAGE_64K; + if (psize == MMU_PAGE_2M) + return H_RPTI_PAGE_2M; + if (psize == MMU_PAGE_1G) + return H_RPTI_PAGE_1G; + return H_RPTI_PAGE_ALL; +} + static inline int mmu_get_ap(int psize) { return mmu_psize_defs[psize].ap; diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index e90c073e437e..43486e773bd6 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -305,7 +305,8 @@ #define H_SCM_UNBIND_ALL 0x3FC #define H_SCM_HEALTH 0x400 #define H_SCM_PERFORMANCE_STATS 0x418 -#define MAX_HCALL_OPCODE H_SCM_PERFORMANCE_STATS +#define H_RPT_INVALIDATE 0x448 +#define MAX_HCALL_OPCODE H_RPT_INVALIDATE /* Scope args for H_SCM_UNBIND_ALL */ #define H_UNBIND_SCOPE_ALL (0x1) @@ -389,6 +390,37 @@ #define PROC_TABLE_RADIX 0x04 #define PROC_TABLE_GTSE 0x01 +/* + * Defines for + * H_RPT_INVALIDATE - Invalidate RPT translation lookaside information. + */ + +/* Type of translation to invalidate (type) */ +#define H_RPTI_TYPE_NESTED 0x0001 /* Invalidate nested guest partition-scope */ +#define H_RPTI_TYPE_TLB 0x0002 /* Invalidate TLB */ +#define H_RPTI_TYPE_PWC 0x0004 /* Invalidate Page Walk Cache */ +/* Invalidate Process Table Entries if H_RPTI_TYPE_NESTED is clear */ +#define H_RPTI_TYPE_PRT 0x0008 +/* Invalidate Partition Table Entries if H_RPTI_TYPE_NESTED is set */ +#define H_RPTI_TYPE_PAT 0x0008 +#define H_RPTI_TYPE_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ + H_RPTI_TYPE_PRT) +#define H_RPTI_TYPE_NESTED_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ + H_RPTI_TYPE_PAT) + +/* Invalidation targets (target) */ +#define H_RPTI_TARGET_CMMU 0x01 /* All virtual processors in the partition */ +#define H_RPTI_TARGET_CMMU_LOCAL 0x02 /* Current virtual processor */ +/* All nest/accelerator agents in use by the partition */ +#define H_RPTI_TARGET_NMMU 0x04 + +/* Page size mask (page sizes) */ +#define H_RPTI_PAGE_4K 0x01 +#define H_RPTI_PAGE_64K 0x02 +#define H_RPTI_PAGE_2M 0x04 +#define H_RPTI_PAGE_1G 0x08 +#define H_RPTI_PAGE_ALL (-1UL) + #ifndef __ASSEMBLY__ #include <linux/types.h> diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h index 4497c8afb573..a184923abd07 100644 --- a/arch/powerpc/include/asm/plpar_wrappers.h +++ b/arch/powerpc/include/asm/plpar_wrappers.h @@ -334,6 +334,49 @@ static inline long plpar_get_cpu_characteristics(struct h_cpu_char_result *p) return rc; } +/* + * Wrapper to H_RPT_INVALIDATE hcall that handles return values appropriately + * + * - Returns H_SUCCESS on success + * - For H_BUSY return value, we retry the hcall. + * - For any other hcall failures, attempt a full flush once before + * resorting to BUG(). + * + * Note: This hcall is expected to fail only very rarely. The correct + * error recovery of killing the process/guest will be eventually + * needed. + */ +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, + u64 page_sizes, u64 start, u64 end) +{ + long rc; + unsigned long all; + + while (true) { + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, type, + page_sizes, start, end); + if (rc == H_BUSY) { + cpu_relax(); + continue; + } else if (rc == H_SUCCESS) + return rc; + + /* Flush request failed, try with a full flush once */ + all = (type & H_RPTI_TYPE_NESTED) ? H_RPTI_TYPE_NESTED_ALL : + H_RPTI_TYPE_ALL; +retry: + rc = plpar_hcall_norets(H_RPT_INVALIDATE, pid, target, + all, page_sizes, 0, -1UL); + if (rc == H_BUSY) { + cpu_relax(); + goto retry; + } else if (rc == H_SUCCESS) + return rc; + + BUG(); + } +} + #else /* !CONFIG_PPC_PSERIES */ static inline long plpar_set_ciabr(unsigned long ciabr) @@ -346,6 +389,13 @@ static inline long plpar_pte_read_4(unsigned long flags, unsigned long ptex, { return 0; } + +static inline long pseries_rpt_invalidate(u32 pid, u64 target, u64 type, + u64 page_sizes, u64 start, u64 end) +{ + return 0; +} + #endif /* CONFIG_PPC_PSERIES */ #endif /* _ASM_POWERPC_PLPAR_WRAPPERS_H */ diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index b5cc9b23cf02..180d8ddcf6e3 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -16,6 +16,7 @@ #include <asm/tlbflush.h> #include <asm/trace.h> #include <asm/cputhreads.h> +#include <asm/plpar_wrappers.h> #define RIC_FLUSH_TLB 0 #define RIC_FLUSH_PWC 1 @@ -694,7 +695,14 @@ void radix__flush_tlb_mm(struct mm_struct *mm) goto local; } - if (cputlb_use_tlbie()) { + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long tgt = H_RPTI_TARGET_CMMU; + + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, + H_RPTI_PAGE_ALL, 0, -1UL); + } else if (cputlb_use_tlbie()) { if (mm_needs_flush_escalation(mm)) _tlbie_pid(pid, RIC_FLUSH_ALL); else @@ -727,7 +735,16 @@ static void __flush_all_mm(struct mm_struct *mm, bool fullmm) goto local; } } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long tgt = H_RPTI_TARGET_CMMU; + unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | + H_RPTI_TYPE_PRT; + + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, type, + H_RPTI_PAGE_ALL, 0, -1UL); + } else if (cputlb_use_tlbie()) _tlbie_pid(pid, RIC_FLUSH_ALL); else _tlbiel_pid_multicast(mm, pid, RIC_FLUSH_ALL); @@ -760,7 +777,19 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr, exit_flush_lazy_tlbs(mm); goto local; } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long tgt, page_sizes, size; + + tgt = H_RPTI_TARGET_CMMU; + page_sizes = psize_to_h_rpti(psize); + size = 1UL << mmu_psize_to_shift(psize); + + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, + page_sizes, vmaddr, + vmaddr + size); + } else if (cputlb_use_tlbie()) _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB); else _tlbiel_va_multicast(mm, vmaddr, pid, psize, RIC_FLUSH_TLB); @@ -810,7 +839,14 @@ static inline void _tlbiel_kernel_broadcast(void) */ void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end) { - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long tgt = H_RPTI_TARGET_CMMU | H_RPTI_TARGET_NMMU; + unsigned long type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | + H_RPTI_TYPE_PRT; + + pseries_rpt_invalidate(0, tgt, type, H_RPTI_PAGE_ALL, + start, end); + } else if (cputlb_use_tlbie()) _tlbie_pid(0, RIC_FLUSH_ALL); else _tlbiel_kernel_broadcast(); @@ -864,7 +900,17 @@ static inline void __radix__flush_tlb_range(struct mm_struct *mm, nr_pages > tlb_local_single_page_flush_ceiling); } - if (full) { + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + unsigned long tgt = H_RPTI_TARGET_CMMU; + unsigned long page_sizes = psize_to_h_rpti(mmu_virtual_psize); + + if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) + page_sizes |= psize_to_h_rpti(MMU_PAGE_2M); + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, H_RPTI_TYPE_TLB, page_sizes, + start, end); + } else if (full) { if (local) { _tlbiel_pid(pid, RIC_FLUSH_TLB); } else { @@ -1046,7 +1092,17 @@ static __always_inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, nr_pages > tlb_local_single_page_flush_ceiling); } - if (full) { + if (!mmu_has_feature(MMU_FTR_GTSE) && !local) { + unsigned long tgt = H_RPTI_TARGET_CMMU; + unsigned long type = H_RPTI_TYPE_TLB; + unsigned long page_sizes = psize_to_h_rpti(psize); + + if (also_pwc) + type |= H_RPTI_TYPE_PWC; + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, type, page_sizes, start, end); + } else if (full) { if (local) { _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); } else { @@ -1111,7 +1167,19 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) exit_flush_lazy_tlbs(mm); goto local; } - if (cputlb_use_tlbie()) + if (!mmu_has_feature(MMU_FTR_GTSE)) { + unsigned long tgt, type, page_sizes; + + tgt = H_RPTI_TARGET_CMMU; + type = H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | + H_RPTI_TYPE_PRT; + page_sizes = psize_to_h_rpti(mmu_virtual_psize); + + if (atomic_read(&mm->context.copros) > 0) + tgt |= H_RPTI_TARGET_NMMU; + pseries_rpt_invalidate(pid, tgt, type, page_sizes, + addr, end); + } else if (cputlb_use_tlbie()) _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true); else _tlbiel_va_range_multicast(mm,