Re: [RFC PATCH v2 20/31] kvx: Add memory management

Add memory management support for kvx, including: cache and tlb management, page fault handling, ioremap/mmap and streaming dma support. Co-developed-by: Clement Leger <clement(a)clement-leger.fr&gt; Signed-off-by: Clement Leger <clement(a)clement-leger.fr&gt; Co-developed-by: Guillaume Thouvenin <gthouvenin(a)kalray.eu&gt; Signed-off-by: Guillaume Thouvenin <gthouvenin(a)kalray.eu&gt; Co-developed-by: Jean-Christophe Pince <jcpince(a)gmail.com&gt; Signed-off-by: Jean-Christophe Pince <jcpince(a)gmail.com&gt; Co-developed-by: Jules Maselbas <jmaselbas(a)kalray.eu&gt; Signed-off-by: Jules Maselbas <jmaselbas(a)kalray.eu&gt; Co-developed-by: Julian Vetter <jvetter(a)kalray.eu&gt; Signed-off-by: Julian Vetter <jvetter(a)kalray.eu&gt; Co-developed-by: Julien Hascoet <jhascoet(a)kalray.eu&gt; Signed-off-by: Julien Hascoet <jhascoet(a)kalray.eu&gt; Co-developed-by: Louis Morhet <lmorhet(a)kalray.eu&gt; Signed-off-by: Louis Morhet <lmorhet(a)kalray.eu&gt; Co-developed-by: Marc Poulhiès <dkm(a)kataplop.net&gt; Signed-off-by: Marc Poulhiès <dkm(a)kataplop.net&gt; Co-developed-by: Marius Gligor <mgligor(a)kalray.eu&gt; Signed-off-by: Marius Gligor <mgligor(a)kalray.eu&gt; Co-developed-by: Vincent Chardon <vincent.chardon(a)elsys-design.com&gt; Signed-off-by: Vincent Chardon <vincent.chardon(a)elsys-design.com&gt; Co-developed-by: Yann Sionneau <ysionneau(a)kalray.eu&gt; Signed-off-by: Yann Sionneau <ysionneau(a)kalray.eu&gt; --- Notes: V1 -> V2: removed L2 cache management arch/kvx/include/asm/cache.h | 43 +++ arch/kvx/include/asm/cacheflush.h | 158 ++++++++++ arch/kvx/include/asm/fixmap.h | 47 +++ arch/kvx/include/asm/hugetlb.h | 36 +++ arch/kvx/include/asm/mem_map.h | 44 +++ arch/kvx/include/asm/mmu.h | 289 ++++++++++++++++++ arch/kvx/include/asm/mmu_context.h | 156 ++++++++++ arch/kvx/include/asm/mmu_stats.h | 38 +++ arch/kvx/include/asm/page.h | 187 ++++++++++++ arch/kvx/include/asm/page_size.h | 29 ++ arch/kvx/include/asm/pgalloc.h | 101 +++++++ arch/kvx/include/asm/pgtable-bits.h | 102 +++++++ arch/kvx/include/asm/pgtable.h | 451 ++++++++++++++++++++++++++++ arch/kvx/include/asm/rm_fw.h | 16 + arch/kvx/include/asm/sparsemem.h | 15 + arch/kvx/include/asm/symbols.h | 16 + arch/kvx/include/asm/tlb.h | 24 ++ arch/kvx/include/asm/tlb_defs.h | 131 ++++++++ arch/kvx/include/asm/tlbflush.h | 58 ++++ arch/kvx/include/asm/vmalloc.h | 10 + arch/kvx/mm/cacheflush.c | 154 ++++++++++ arch/kvx/mm/dma-mapping.c | 85 ++++++ arch/kvx/mm/extable.c | 24 ++ arch/kvx/mm/fault.c | 264 ++++++++++++++++ arch/kvx/mm/init.c | 277 +++++++++++++++++ arch/kvx/mm/mmap.c | 31 ++ arch/kvx/mm/mmu.c | 202 +++++++++++++ arch/kvx/mm/mmu_stats.c | 94 ++++++ arch/kvx/mm/tlb.c | 433 ++++++++++++++++++++++++++ 29 files changed, 3515 insertions(+) create mode 100644 arch/kvx/include/asm/cache.h create mode 100644 arch/kvx/include/asm/cacheflush.h create mode 100644 arch/kvx/include/asm/fixmap.h create mode 100644 arch/kvx/include/asm/hugetlb.h create mode 100644 arch/kvx/include/asm/mem_map.h create mode 100644 arch/kvx/include/asm/mmu.h create mode 100644 arch/kvx/include/asm/mmu_context.h create mode 100644 arch/kvx/include/asm/mmu_stats.h create mode 100644 arch/kvx/include/asm/page.h create mode 100644 arch/kvx/include/asm/page_size.h create mode 100644 arch/kvx/include/asm/pgalloc.h create mode 100644 arch/kvx/include/asm/pgtable-bits.h create mode 100644 arch/kvx/include/asm/pgtable.h create mode 100644 arch/kvx/include/asm/rm_fw.h create mode 100644 arch/kvx/include/asm/sparsemem.h create mode 100644 arch/kvx/include/asm/symbols.h create mode 100644 arch/kvx/include/asm/tlb.h create mode 100644 arch/kvx/include/asm/tlb_defs.h create mode 100644 arch/kvx/include/asm/tlbflush.h create mode 100644 arch/kvx/include/asm/vmalloc.h create mode 100644 arch/kvx/mm/cacheflush.c create mode 100644 arch/kvx/mm/dma-mapping.c create mode 100644 arch/kvx/mm/extable.c create mode 100644 arch/kvx/mm/fault.c create mode 100644 arch/kvx/mm/init.c create mode 100644 arch/kvx/mm/mmap.c create mode 100644 arch/kvx/mm/mmu.c create mode 100644 arch/kvx/mm/mmu_stats.c create mode 100644 arch/kvx/mm/tlb.c

diff --git a/arch/kvx/include/asm/mmu.h b/arch/kvx/include/asm/mmu.h new file mode 100644 index 000000000000..09f3fdd66a34 --- /dev/null +++ b/arch/kvx/include/asm/mmu.h @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Guillaume Thouvenin + * Clement Leger + * Marc Poulhiès + */ + +#ifndef _ASM_KVX_MMU_H +#define _ASM_KVX_MMU_H + +#include <linux/bug.h> +#include <linux/types.h> +#include <linux/threads.h> + +#include <asm/page.h> +#include <asm/sfr.h> +#include <asm/page.h> +#include <asm/pgtable-bits.h> +#include <asm/tlb_defs.h> + +/* Virtual addresses can use at most 41 bits */ +#define MMU_VIRT_BITS 41 + +/* + * See Documentation/kvx/kvx-mmu.txt for details about the division of the + * virtual memory space. + */ +#if defined(CONFIG_KVX_4K_PAGES) +#define MMU_USR_ADDR_BITS 39 +#else +#error "Only 4Ko page size is supported at this time" +#endif + +typedef struct mm_context { + unsigned long end_brk; + unsigned long asn[NR_CPUS]; + unsigned long sigpage; +} mm_context_t; + +struct __packed tlb_entry_low { + unsigned int es:2; /* Entry Status */ + unsigned int cp:2; /* Cache Policy */ + unsigned int pa:4; /* Protection Attributes */ + unsigned int r:2; /* Reserved */ + unsigned int ps:2; /* Page Size */ + unsigned int fn:28; /* Frame Number */ +}; + +struct __packed tlb_entry_high { + unsigned int asn:9; /* Address Space Number */ + unsigned int g:1; /* Global Indicator */ + unsigned int vs:2; /* Virtual Space */ + unsigned int pn:29; /* Page Number */ +}; + +struct kvx_tlb_format { + union { + struct tlb_entry_low tel; + uint64_t tel_val; + }; + union { + struct tlb_entry_high teh; + uint64_t teh_val; + }; +};

+ +#define KVX_EMPTY_TLB_ENTRY { .tel_val = 0x0, .teh_val = 0x0 } + +/* Bit [0:39] of the TLB format corresponds to TLB Entry low */ +/* Bit [40:80] of the TLB format corresponds to the TLB Entry high */ +#define kvx_mmu_set_tlb_entry(tlbf) do { \ + kvx_sfr_set(TEL, (uint64_t) tlbf.tel_val); \ + kvx_sfr_set(TEH, (uint64_t) tlbf.teh_val); \ +} while (0) + +#define kvx_mmu_get_tlb_entry(tlbf) do { \ + tlbf.tel_val = kvx_sfr_get(TEL); \ + tlbf.teh_val = kvx_sfr_get(TEH); \ +} while (0) + +/* Use kvx_mmc_ to read a field from MMC value passed as parameter */ +#define __kvx_mmc(mmc_reg, field) \ + kvx_sfr_field_val(mmc_reg, MMC, field) +

diff --git a/arch/kvx/include/asm/mmu_context.h b/arch/kvx/include/asm/mmu_context.h new file mode 100644 index 000000000000..39fa92f1506b --- /dev/null +++ b/arch/kvx/include/asm/mmu_context.h @@ -0,0 +1,156 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + * Guillaume Thouvenin + */ + +#ifndef __ASM_KVX_MMU_CONTEXT_H +#define __ASM_KVX_MMU_CONTEXT_H + +/* + * Management of the Address Space Number: + * Coolidge architecture provides a 9-bit ASN to tag TLB entries. This can be + * used to allow several entries with the same virtual address (so from + * different process) to be in the TLB at the same time. That means that won't + * necessarily flush the TLB when a context switch occurs and so it will + * improve performances. + */ +#include <linux/smp.h> + +#include <asm/mmu.h> +#include <asm/sfr_defs.h> +#include <asm/tlbflush.h> + +#include <asm-generic/mm_hooks.h>

+/** + * Redefining the generic hooks that are: + * - activate_mm + * - deactivate_mm + * - enter_lazy_tlb + * - init_new_context + * - destroy_context + * - switch_mm + */

+ +#define activate_mm(prev, next) switch_mm((prev), (next), NULL) +#define deactivate_mm(tsk, mm) do { } while (0) +#define enter_lazy_tlb(mm, tsk) do { } while (0) +

+#endif /* __ASM_KVX_MMU_CONTEXT_H */ diff --git a/arch/kvx/include/asm/mmu_stats.h b/arch/kvx/include/asm/mmu_stats.h new file mode 100644 index 000000000000..999352dbc1ce

--- /dev/null +++ b/arch/kvx/include/asm/mmu_stats.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + */ + +#ifndef _ASM_KVX_MMU_STATS_H +#define _ASM_KVX_MMU_STATS_H + +#ifdef CONFIG_KVX_MMU_STATS +#include <linux/percpu.h> + +struct mmu_refill_stats { + unsigned long count; + unsigned long total; + unsigned long min; + unsigned long max; +}; + +enum mmu_refill_type { + MMU_REFILL_TYPE_USER, + MMU_REFILL_TYPE_KERNEL, + MMU_REFILL_TYPE_KERNEL_DIRECT, + MMU_REFILL_TYPE_COUNT, +}; + +struct mmu_stats { + struct mmu_refill_stats refill[MMU_REFILL_TYPE_COUNT]; + /* keep these fields ordered this way for assembly */ + unsigned long cycles_between_refill; + unsigned long last_refill; + unsigned long tlb_flush_all; +}; + +DECLARE_PER_CPU(struct mmu_stats, mmu_stats); +#endif + +#endif /* _ASM_KVX_MMU_STATS_H */

diff --git a/arch/kvx/include/asm/pgalloc.h b/arch/kvx/include/asm/pgalloc.h new file mode 100644 index 000000000000..0e654dd1a072 --- /dev/null +++ b/arch/kvx/include/asm/pgalloc.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Guillaume Thouvenin + * Clement Leger + */ + +#ifndef _ASM_KVX_PGALLOC_H +#define _ASM_KVX_PGALLOC_H + +#include <linux/mm.h> +#include <asm/tlb.h> + +#define __HAVE_ARCH_PGD_FREE +#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ + +static inline void check_pgt_cache(void) +{ + /* + * check_pgt_cache() is called to check watermarks from counters that + * computes the number of pages allocated by cached allocation functions + * pmd_alloc_one_fast() and pte_alloc_one_fast(). + * Currently we just skip this test. + */

+} + +/** + * PGD + */

+ +static inline void +pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + free_pages((unsigned long) pgd, PAGES_PER_PGD); +} + +static inline +pgd_t *pgd_alloc(struct mm_struct *mm) +{ + pgd_t *pgd; + + pgd = (pgd_t *) __get_free_pages(GFP_KERNEL, PAGES_PER_PGD); + if (unlikely(pgd == NULL)) + return NULL; + + memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); + + /* Copy kernel mappings */ + memcpy(pgd + USER_PTRS_PER_PGD, + init_mm.pgd + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + + return pgd; +} + +/** + * PUD + */ + +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + unsigned long pfn = virt_to_pfn(pmd); + + set_pud(pud, __pud((unsigned long)pfn << PAGE_SHIFT)); +} + +/** + * PMD + */ + +static inline void pmd_populate_kernel(struct mm_struct *mm, + pmd_t *pmd, pte_t *pte) +{ + unsigned long pfn = virt_to_pfn(pte); + + set_pmd(pmd, __pmd((unsigned long)pfn << PAGE_SHIFT)); +} + +static inline void pmd_populate(struct mm_struct *mm, + pmd_t *pmd, pgtable_t pte) +{ + unsigned long pfn = virt_to_pfn(page_address(pte)); + + set_pmd(pmd, __pmd((unsigned long)pfn << PAGE_SHIFT)); +} + +#if CONFIG_PGTABLE_LEVELS > 2 +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) +#endif /* CONFIG_PGTABLE_LEVELS > 2 */ + +/** + * PTE + */ + +#define __pte_free_tlb(tlb, pte, buf) \ +do { \ + pgtable_pte_page_dtor(pte); \ + tlb_remove_page((tlb), pte); \ +} while (0) + +#endif /* _ASM_KVX_PGALLOC_H */

diff --git a/arch/kvx/include/asm/pgtable.h b/arch/kvx/include/asm/pgtable.h new file mode 100644 index 000000000000..9e36db4d98a7 --- /dev/null +++ b/arch/kvx/include/asm/pgtable.h @@ -0,0 +1,451 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Guillaume Thouvenin + * Clement Leger + * Marius Gligor + * Yann Sionneau + */ + +#ifndef _ASM_KVX_PGTABLE_H +#define _ASM_KVX_PGTABLE_H +

+ +/* + * PGD definitions: + * - pgd_ERROR + */

+#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %016lx.\n", __FILE__, __LINE__, pgd_val(e)) + +/* + * PUD + * + * As we manage a three level page table the call to set_pud is used to fill + * PGD. + */ +static inline void set_pud(pud_t *pudp, pud_t pmd) +{ + *pudp = pmd; +} + +static inline int pud_none(pud_t pud) +{ + return pud_val(pud) == 0; +} + +static inline int pud_bad(pud_t pud) +{ + return pud_none(pud); +} +static inline int pud_present(pud_t pud) +{ + return pud_val(pud) != 0; +} + +static inline void pud_clear(pud_t *pud) +{ + set_pud(pud, __pud(0)); +} + +/* + * PMD definitions: + * - set_pmd + * - pmd_present + * - pmd_none + * - pmd_bad + * - pmd_clear + * - pmd_page + */

+ +static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} + +/* Returns 1 if entry is present */ +static inline int pmd_present(pmd_t pmd) +{ + return pmd_val(pmd) != 0; +} + +/* Returns 1 if the corresponding entry has the value 0 */ +static inline int pmd_none(pmd_t pmd) +{ + return pmd_val(pmd) == 0; +} + +/* Used to check that a page middle directory entry is valid */ +static inline int pmd_bad(pmd_t pmd) +{ + return pmd_none(pmd); +} + +/* Clears the entry to prevent process to use the linear address that + * mapped it. + */ +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pmd(pmdp, __pmd(0)); +} + +/* + * Returns the address of the descriptor of the page table referred by the + * PMD entry. + */ +static inline struct page *pmd_page(pmd_t pmd) +{ + if (pmd_val(pmd) & _PAGE_HUGE) + return pfn_to_page( + (pmd_val(pmd) & KVX_PFN_MASK) >> KVX_PFN_SHIFT); + + return pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT); +} + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + return (pmd_t *)pfn_to_virt(pud_val(pud) >> PAGE_SHIFT); +} + +static inline struct page *pud_page(pud_t pud) +{ + return pfn_to_page(pud_val(pud) >> PAGE_SHIFT); +} + +/* + * PTE definitions: + * - set_pte + * - set_pte_at + * - pte_clear + * - pte_page + * - pte_pfn + * - pte_present + * - pte_none + * - pte_write + * - pte_dirty + * - pte_young + * - pte_special + * - pte_mkdirty + * - pte_mkwrite + * - pte_mkclean + * - pte_mkyoung + * - pte_mkold + * - pte_mkspecial + * - pte_wrprotect + */

+ +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + *ptep = pteval; +} + +static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pteval) +{ + set_pte(ptep, pteval); +} + +#define pte_clear(mm, addr, ptep) set_pte(ptep, __pte(0)) + +/* Constructs a page table entry */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) +{ + return __pte(((pfn << KVX_PFN_SHIFT) & KVX_PFN_MASK) | + pgprot_val(prot)); +} + +/* Builds a page table entry by combining a page descriptor and a group of + * access rights. + */ +#define mk_pte(page, prot) (pfn_pte(page_to_pfn(page), prot)) + +/* Modifies page access rights */ +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); +} + +#define pte_page(x) pfn_to_page(pte_pfn(x)) + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + return (unsigned long)pfn_to_virt(pmd_val(pmd) >> PAGE_SHIFT); +} + +/* Yields the page frame number (PFN) of a page table entry */ +static inline unsigned long pte_pfn(pte_t pte) +{ + return ((pte_val(pte) & KVX_PFN_MASK) >> KVX_PFN_SHIFT); +} + +static inline int pte_present(pte_t pte) +{ + return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_NONE)); +} + +static inline int pte_none(pte_t pte) +{ + return (pte_val(pte) == 0); +} + +static inline int pte_write(pte_t pte) +{ + return pte_val(pte) & _PAGE_WRITE; +} + +static inline int pte_dirty(pte_t pte) +{ + return pte_val(pte) & _PAGE_DIRTY; +} + +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & _PAGE_ACCESSED; +} + +static inline int pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +static inline int pte_huge(pte_t pte) +{ + return pte_val(pte) & _PAGE_HUGE; +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_DIRTY); +} + +static inline pte_t pte_mkwrite(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_WRITE); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_DIRTY)); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_ACCESSED); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_ACCESSED)); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SPECIAL); +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~(_PAGE_WRITE)); +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_HUGE); +} + +static inline pte_t pte_of_pmd(pmd_t pmd) +{ + return __pte(pmd_val(pmd)); +} + +#define pmd_pfn(pmd) pte_pfn(pte_of_pmd(pmd)) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE

+ +#define pmdp_establish pmdp_establish +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + return __pmd(xchg(&pmd_val(*pmdp), pmd_val(pmd))); +} + +static inline int pmd_trans_huge(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_HUGE); +} + +static inline pmd_t pmd_of_pte(pte_t pte) +{ + return __pmd(pte_val(pte)); +} + + +#define pmd_mkclean(pmd) pmd_of_pte(pte_mkclean(pte_of_pmd(pmd))) +#define pmd_mkdirty(pmd) pmd_of_pte(pte_mkdirty(pte_of_pmd(pmd))) +#define pmd_mkold(pmd) pmd_of_pte(pte_mkold(pte_of_pmd(pmd))) +#define pmd_mkwrite(pmd) pmd_of_pte(pte_mkwrite(pte_of_pmd(pmd))) +#define pmd_mkyoung(pmd) pmd_of_pte(pte_mkyoung(pte_of_pmd(pmd))) +#define pmd_modify(pmd, prot) pmd_of_pte(pte_modify(pte_of_pmd(pmd), prot)) +#define pmd_wrprotect(pmd) pmd_of_pte(pte_wrprotect(pte_of_pmd(pmd))) + +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + /* Create a huge page in PMD implies a size of 2 MB */ + return __pmd(pmd_val(pmd) | + _PAGE_HUGE | (TLB_PS_2M << KVX_PAGE_SZ_SHIFT)); +} + +static inline pmd_t pmd_mkinvalid(pmd_t pmd) +{ + pmd_val(pmd) &= ~(_PAGE_PRESENT); + + return pmd; +} + +#define pmd_dirty(pmd) pte_dirty(pte_of_pmd(pmd)) +#define pmd_write(pmd) pte_write(pte_of_pmd(pmd)) +#define pmd_young(pmd) pte_young(pte_of_pmd(pmd)) + +#define mk_pmd(page, prot) pmd_of_pte(mk_pte(page, prot)) + +static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) +{ + return __pmd(((pfn << KVX_PFN_SHIFT) & KVX_PFN_MASK) | + pgprot_val(prot)); +} + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} + +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#endif /* _ASM_KVX_PGTABLE_H */ diff --git a/arch/kvx/include/asm/rm_fw.h b/arch/kvx/include/asm/rm_fw.h new file mode 100644 index 000000000000..f89bdd5915ed --- /dev/null +++ b/arch/kvx/include/asm/rm_fw.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + */ + +#ifndef _ASM_KVX_RM_FW_H +#define _ASM_KVX_RM_FW_H + +#include <linux/sizes.h> + +#define KVX_RM_ID 16 + +#define RM_FIRMWARE_REGS_SIZE (SZ_4K) + +#endif /* _ASM_KVX_RM_FW_H */ diff --git a/arch/kvx/include/asm/sparsemem.h b/arch/kvx/include/asm/sparsemem.h new file mode 100644 index 000000000000..2f35743f20fb --- /dev/null +++ b/arch/kvx/include/asm/sparsemem.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + */ + +#ifndef _ASM_KVX_SPARSEMEM_H +#define _ASM_KVX_SPARSEMEM_H

+ +#ifdef CONFIG_SPARSEMEM +#define MAX_PHYSMEM_BITS 40 +#define SECTION_SIZE_BITS 30 +#endif /* CONFIG_SPARSEMEM */ + +#endif /* _ASM_KVX_SPARSEMEM_H */ diff --git a/arch/kvx/include/asm/symbols.h b/arch/kvx/include/asm/symbols.h new file mode 100644 index 000000000000..a53c1607979f --- /dev/null +++ b/arch/kvx/include/asm/symbols.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + */ + +#ifndef _ASM_KVX_SYMBOLS_H +#define _ASM_KVX_SYMBOLS_H + +/* Symbols to patch TLB refill handler */ +extern char kvx_perf_tlb_refill[], kvx_std_tlb_refill[]; + +/* Entry point of the ELF, used to start other PEs in SMP */ +extern int kvx_start[]; + +#endif diff --git a/arch/kvx/include/asm/tlb.h b/arch/kvx/include/asm/tlb.h new file mode 100644 index 000000000000..190b682e1819 --- /dev/null +++ b/arch/kvx/include/asm/tlb.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Guillaume Thouvenin + * Clement Leger + */ + +#ifndef _ASM_KVX_TLB_H +#define _ASM_KVX_TLB_H + +struct mmu_gather; + +static void tlb_flush(struct mmu_gather *tlb); + +int clear_ltlb_entry(unsigned long vaddr); + +#include <asm-generic/tlb.h> + +static inline unsigned int pgprot_cache_policy(unsigned long flags) +{ + return (flags & KVX_PAGE_CP_MASK) >> KVX_PAGE_CP_SHIFT; +} + +#endif /* _ASM_KVX_TLB_H */ diff --git a/arch/kvx/include/asm/tlb_defs.h b/arch/kvx/include/asm/tlb_defs.h new file mode 100644 index 000000000000..3f5b29cd529e --- /dev/null +++ b/arch/kvx/include/asm/tlb_defs.h

@@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + * Julian Vetter + * Guillaume Thouvenin + * Marius Gligor + */ + +#ifndef _ASM_KVX_TLB_DEFS_H +#define _ASM_KVX_TLB_DEFS_H +

diff --git a/arch/kvx/mm/init.c b/arch/kvx/mm/init.c new file mode 100644 index 000000000000..bac34bc09eb5 --- /dev/null +++ b/arch/kvx/mm/init.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + * Guillaume Thouvenin + */ + +/* Memblock header depends on types.h but does not include it ! */ +#include <linux/types.h> +#include <linux/memblock.h> +#include <linux/mmzone.h> +#include <linux/of_fdt.h> +#include <linux/sched.h> +#include <linux/sizes.h> +#include <linux/init.h> +#include <linux/initrd.h> +#include <linux/pfn.h> +#include <linux/mm.h> + +#include <asm/sections.h> +#include <asm/tlb_defs.h> +#include <asm/tlbflush.h> +#include <asm/fixmap.h> +#include <asm/page.h> + +/* + * On kvx, memory map contains the first 2G of DDR being aliased. + * Full contiguous DDR is located at @[4G - 68G]. + * However, to access this DDR in 32bit mode, the first 2G of DDR are + * mirrored from 4G to 2G. + * These first 2G are accessible from all DMAs (included 32 bits one). + * + * Hence, the memory map is the following: + * + * (68G) 0x1100000000-> +-------------+ + * | | + * 66G |(ZONE_NORMAL)| + * | | + * (6G) 0x180000000-> +-------------+ + * | | + * 2G |(ZONE_DMA32) | + * | | + * (4G) 0x100000000-> +-------------+ +--+ + * | | | + * 2G | (Alias) | | 2G Alias + * | | | + * (2G) 0x80000000-> +-------------+ <--+ + * + * The translation of 64 bits -> 32 bits can then be done using dma-ranges property + * in device-trees. + */ + +#define DDR_64BIT_START (4ULL * SZ_1G) +#define DDR_32BIT_ALIAS_SIZE (2ULL * SZ_1G) + +#define MAX_DMA32_PFN PHYS_PFN(DDR_64BIT_START + DDR_32BIT_ALIAS_SIZE) + +pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned_bss; + +/* + * empty_zero_page is a special page that is used for zero-initialized data and + * COW. + */ +struct page *empty_zero_page; +EXPORT_SYMBOL(empty_zero_page); + +extern char _start[]; +extern char __kernel_smem_code_start[]; +extern char __kernel_smem_code_end[]; + +struct kernel_section { + phys_addr_t start; + phys_addr_t end; +}; + +struct kernel_section kernel_sections[] = { + { + .start = (phys_addr_t)__kernel_smem_code_start, + .end = (phys_addr_t)__kernel_smem_code_end + }, + { + .start = __pa(_start), + .end = __pa(_end) + } +}; + +static void __init zone_sizes_init(void) +{ + unsigned long zones_size[MAX_NR_ZONES]; + + memset(zones_size, 0, sizeof(zones_size)); + + zones_size[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); + zones_size[ZONE_NORMAL] = max_low_pfn; + + free_area_init(zones_size); +} + +#ifdef CONFIG_BLK_DEV_INITRD +static void __init setup_initrd(void) +{ + u64 base = phys_initrd_start; + u64 end = phys_initrd_start + phys_initrd_size; + + if (phys_initrd_size == 0) { + pr_info("initrd not found or empty"); + return; + } + + if (base < memblock_start_of_DRAM() || end > memblock_end_of_DRAM()) { + pr_err("initrd not in accessible memory, disabling it"); + phys_initrd_size = 0; + return; + } + + pr_info("initrd: 0x%llx - 0x%llx\n", base, end); + + memblock_reserve(phys_initrd_start, phys_initrd_size); + + /* the generic initrd code expects virtual addresses */ + initrd_start = (unsigned long) __va(base); + initrd_end = initrd_start + phys_initrd_size; +} +#endif + +static phys_addr_t memory_limit = PHYS_ADDR_MAX; + +static int __init early_mem(char *p) +{ + if (!p) + return 1; + + memory_limit = memparse(p, &p) & PAGE_MASK; + pr_notice("Memory limited to %lldMB\n", memory_limit >> 20); + + return 0; +} +early_param("mem", early_mem); + +static void __init setup_bootmem(void) +{ + phys_addr_t kernel_start, kernel_end; + phys_addr_t start, end = 0; + u64 i; + + init_mm.start_code = (unsigned long)_stext; + init_mm.end_code = (unsigned long)_etext; + init_mm.end_data = (unsigned long)_edata; + init_mm.brk = (unsigned long)_end; + + for (i = 0; i < ARRAY_SIZE(kernel_sections); i++) { + kernel_start = kernel_sections[i].start; + kernel_end = kernel_sections[i].end; + + memblock_reserve(kernel_start, kernel_end - kernel_start); + } + + for_each_mem_range(i, &start, &end) { + pr_info("%15s: memory : 0x%lx - 0x%lx\n", __func__, + (unsigned long)start, + (unsigned long)end); + } + + /* min_low_pfn is the lowest PFN available in the system */ + min_low_pfn = PFN_UP(memblock_start_of_DRAM()); + + /* max_low_pfn indicates the end if NORMAL zone */ + max_low_pfn = PFN_DOWN(memblock_end_of_DRAM());

+ + /* Set the maximum number of pages in the system */ + set_max_mapnr(max_low_pfn - min_low_pfn); + +#ifdef CONFIG_BLK_DEV_INITRD + setup_initrd(); +#endif + + if (memory_limit != PHYS_ADDR_MAX) + memblock_mem_limit_remove_map(memory_limit);

+ + /* Don't reserve the device tree if its builtin */ + if (!is_kernel_rodata((unsigned long) initial_boot_params)) + early_init_fdt_reserve_self(); + early_init_fdt_scan_reserved_mem(); + + memblock_allow_resize(); + memblock_dump_all(); +} + +static pmd_t fixmap_pmd[PTRS_PER_PMD] __page_aligned_bss __maybe_unused; +static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss __maybe_unused; + +void __init early_fixmap_init(void) +{ + unsigned long vaddr; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + /* + * Fixed mappings: + */ + vaddr = __fix_to_virt(__end_of_fixed_addresses - 1); + pgd = pgd_offset_pgd(swapper_pg_dir, vaddr); + set_pgd(pgd, __pgd(__pa_symbol(fixmap_pmd))); + + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); + pmd = pmd_offset(pud, vaddr); + set_pmd(pmd, __pmd(__pa_symbol(fixmap_pte))); +} + +void __init setup_arch_memory(void) +{ + setup_bootmem(); + sparse_init(); + zone_sizes_init(); +} + +void __init mem_init(void) +{ + memblock_free_all(); + + /* allocate the zero page */ + empty_zero_page = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!empty_zero_page) + panic("Failed to allocate the empty_zero_page"); +} + +void free_initmem(void) +{ +#ifdef CONFIG_POISON_INITMEM + free_initmem_default(0x0); +#else + free_initmem_default(-1); +#endif

+} + +void __set_fixmap(enum fixed_addresses idx, + phys_addr_t phys, pgprot_t flags) +{ + unsigned long addr = __fix_to_virt(idx); + pte_t *pte; + + + BUG_ON(idx >= __end_of_fixed_addresses); + + pte = &fixmap_pte[pte_index(addr)]; + + if (pgprot_val(flags)) { + set_pte(pte, pfn_pte(phys_to_pfn(phys), flags)); + } else { + /* Remove the fixmap */ + pte_clear(&init_mm, addr, pte); + } + local_flush_tlb_kernel_range(addr, addr + PAGE_SIZE); +} + +static const pgprot_t protection_map[16] = { + [VM_NONE] = PAGE_NONE, + [VM_READ] = PAGE_READ, + [VM_WRITE] = PAGE_READ, + [VM_WRITE | VM_READ] = PAGE_READ, + [VM_EXEC] = PAGE_READ_EXEC, + [VM_EXEC | VM_READ] = PAGE_READ_EXEC, + [VM_EXEC | VM_WRITE] = PAGE_READ_EXEC, + [VM_EXEC | VM_WRITE | VM_READ] = PAGE_READ_EXEC, + [VM_SHARED] = PAGE_NONE, + [VM_SHARED | VM_READ] = PAGE_READ, + [VM_SHARED | VM_WRITE] = PAGE_READ_WRITE, + [VM_SHARED | VM_WRITE | VM_READ] = PAGE_READ_WRITE, + [VM_SHARED | VM_EXEC] = PAGE_READ_EXEC, + [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READ_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_READ_WRITE_EXEC, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_READ_WRITE_EXEC +}; +DECLARE_VM_GET_PAGE_PROT diff --git a/arch/kvx/mm/mmap.c b/arch/kvx/mm/mmap.c new file mode 100644 index 000000000000..a2225db64438 --- /dev/null +++ b/arch/kvx/mm/mmap.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * derived from arch/arm64/mm/mmap.c + * + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + */ + +#ifdef CONFIG_STRICT_DEVMEM + +#include <linux/mm.h> +#include <linux/ioport.h> + +#include <asm/page.h> + +/* + * devmem_is_allowed() checks to see if /dev/mem access to a certain address + * is valid. The argument is a physical page number. We mimic x86 here by + * disallowing access to system RAM as well as device-exclusive MMIO regions. + * This effectively disable read()/write() on /dev/mem. + */ +int devmem_is_allowed(unsigned long pfn) +{ + if (iomem_is_exclusive(pfn << PAGE_SHIFT)) + return 0; + if (!page_is_ram(pfn))

+ return 1; + return 0; +} + +#endif diff --git a/arch/kvx/mm/mmu.c b/arch/kvx/mm/mmu.c new file mode 100644 index 000000000000..9cb11bd2dfdf --- /dev/null +++ b/arch/kvx/mm/mmu.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2017-2023 Kalray Inc. + * Author(s): Clement Leger + * Guillaume Thouvenin + * Vincent Chardon + * Jules Maselbas + */ + +#include <linux/cache.h> +#include <linux/types.h> +#include <linux/irqflags.h> +#include <linux/printk.h> +#include <linux/percpu.h> +#include <linux/kernel.h> +#include <linux/spinlock.h> +#include <linux/spinlock_types.h> + +#include <asm/mmu.h> +#include <asm/tlb.h> +#include <asm/page_size.h> +#include <asm/mmu_context.h> + +#define DUMP_LTLB 0 +#define DUMP_JTLB 1 + +DEFINE_PER_CPU_ALIGNED(uint8_t[MMU_JTLB_SETS], jtlb_current_set_way); +static struct kvx_tlb_format ltlb_entries[MMU_LTLB_WAYS]; +static unsigned long ltlb_entries_bmp; + +static int kvx_mmu_ltlb_overlaps(struct kvx_tlb_format tlbe) +{ + int bit = LTLB_ENTRY_FIXED_COUNT; + + for_each_set_bit_from(bit, &ltlb_entries_bmp, MMU_LTLB_WAYS) { + if (tlb_entry_overlaps(tlbe, ltlb_entries[bit])) + return 1; + } + + return 0; +} + +/** + * kvx_mmu_ltlb_add_entry - Add a kernel entry in the LTLB + * + * In order to lock some entries in the LTLB and be always mapped, this + * function can be called with a physical address, a virtual address and + * protection attribute to add an entry into the LTLB. This is mainly for + * performances since there won't be any NOMAPPING traps for these pages. + * + * @vaddr: Virtual address for the entry (must be aligned according to tlb_ps) + * @paddr: Physical address for the entry (must be aligned according to tlb_ps) + * @flags: Protection attributes + * @tlb_ps: Page size attribute for TLB (TLB_PS_*) + */ +void kvx_mmu_ltlb_add_entry(unsigned long vaddr, phys_addr_t paddr, + pgprot_t flags, unsigned long tlb_ps) +{ + unsigned int cp; + unsigned int idx; + unsigned long irqflags; + struct kvx_tlb_format tlbe; + u64 page_size = BIT(get_page_size_shift(tlb_ps)); + + BUG_ON(!IS_ALIGNED(vaddr, page_size) || !IS_ALIGNED(paddr, page_size)); + + cp = pgprot_cache_policy(pgprot_val(flags)); + + tlbe = tlb_mk_entry( + (void *) paddr, + (void *) vaddr,

+ tlb_ps, + TLB_G_GLOBAL, + TLB_PA_NA_RW, + cp, + 0, + TLB_ES_A_MODIFIED);

+ + local_irq_save(irqflags); + + if (IS_ENABLED(CONFIG_KVX_DEBUG_TLB_WRITE) && + kvx_mmu_ltlb_overlaps(tlbe)) + panic("VA %lx overlaps with an existing LTLB mapping", vaddr); + + idx = find_next_zero_bit(&ltlb_entries_bmp, MMU_LTLB_WAYS, + LTLB_ENTRY_FIXED_COUNT); + /* This should never happen */ + BUG_ON(idx >= MMU_LTLB_WAYS); + __set_bit(idx, &ltlb_entries_bmp); + ltlb_entries[idx] = tlbe; + kvx_mmu_add_entry(MMC_SB_LTLB, idx, tlbe); + + if (kvx_mmc_error(kvx_sfr_get(MMC))) + panic("Failed to write entry to the LTLB"); + + local_irq_restore(irqflags); +} + +void kvx_mmu_ltlb_remove_entry(unsigned long vaddr) +{ + int ret, bit = LTLB_ENTRY_FIXED_COUNT; + struct kvx_tlb_format tlbe; + + for_each_set_bit_from(bit, &ltlb_entries_bmp, MMU_LTLB_WAYS) { + tlbe = ltlb_entries[bit]; + if (tlb_entry_match_addr(tlbe, vaddr)) { + __clear_bit(bit, &ltlb_entries_bmp); + break; + } + } + + if (bit == MMU_LTLB_WAYS) + panic("Trying to remove non-existent LTLB entry for addr %lx\n", + vaddr); + + ret = clear_ltlb_entry(vaddr); + if (ret) + panic("Failed to remove LTLB entry for addr %lx\n", vaddr); +} + +/** + * kvx_mmu_jtlb_add_entry - Add an entry into JTLB + * + * JTLB is used both for kernel and user entries. + * + * @address: Virtual address for the entry (must be aligned according to tlb_ps) + * @ptep: pte entry pointer + * @asn: ASN (if pte entry is not global) + */ +void kvx_mmu_jtlb_add_entry(unsigned long address, pte_t *ptep, + unsigned int asn) +{ + unsigned int shifted_addr, set, way; + unsigned long flags, pte_val; + struct kvx_tlb_format tlbe; + unsigned int pa, cp, ps; + phys_addr_t pfn; + + pte_val = pte_val(*ptep); + + pfn = (phys_addr_t)pte_pfn(*ptep); + + asn &= MM_CTXT_ASN_MASK; + + /* Set page as accessed */ + pte_val(*ptep) |= _PAGE_ACCESSED; + + BUILD_BUG_ON(KVX_PAGE_SZ_SHIFT != KVX_SFR_TEL_PS_SHIFT); + + ps = (pte_val & KVX_PAGE_SZ_MASK) >> KVX_PAGE_SZ_SHIFT; + pa = get_page_access_perms(KVX_ACCESS_PERMS_INDEX(pte_val)); + cp = pgprot_cache_policy(pte_val); + + tlbe = tlb_mk_entry( + (void *)pfn_to_phys(pfn), + (void *)address, + ps, + (pte_val & _PAGE_GLOBAL) ? TLB_G_GLOBAL : TLB_G_USE_ASN, + pa, + cp, + asn, + TLB_ES_A_MODIFIED);

+ + shifted_addr = address >> get_page_size_shift(ps); + set = shifted_addr & MMU_JTLB_SET_MASK; + + local_irq_save(flags); + + if (IS_ENABLED(CONFIG_KVX_DEBUG_TLB_WRITE) && + kvx_mmu_ltlb_overlaps(tlbe)) + panic("VA %lx overlaps with an existing LTLB mapping", address); + + way = get_cpu_var(jtlb_current_set_way)[set]++; + put_cpu_var(jtlb_current_set_way); + + way &= MMU_JTLB_WAY_MASK; + + kvx_mmu_add_entry(MMC_SB_JTLB, way, tlbe); + + if (IS_ENABLED(CONFIG_KVX_DEBUG_TLB_WRITE) && + kvx_mmc_error(kvx_sfr_get(MMC))) + panic("Failed to write entry to the JTLB (in update_mmu_cache)"); + + local_irq_restore(flags); +} + +void __init kvx_mmu_early_setup(void) +{ + int bit; + struct kvx_tlb_format tlbe; + + kvx_mmu_remove_ltlb_entry(LTLB_ENTRY_EARLY_SMEM); + + if (raw_smp_processor_id() != 0) { + /* Apply existing ltlb entries starting from first one free */ + bit = LTLB_ENTRY_FIXED_COUNT; + for_each_set_bit_from(bit, &ltlb_entries_bmp, MMU_LTLB_WAYS) { + tlbe = ltlb_entries[bit]; + kvx_mmu_add_entry(MMC_SB_LTLB, bit, tlbe); + } + } +}