diff --git a/hal/aarch64/pmap.c b/hal/aarch64/pmap.c index 9734a36be..39cad5415 100644 --- a/hal/aarch64/pmap.c +++ b/hal/aarch64/pmap.c @@ -273,7 +273,7 @@ static void _pmap_cacheOpAfterChange(descr_t newEntry, ptr_t vaddr, unsigned int /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { pmap->ttl1 = vaddr; pmap->addr = p->addr; diff --git a/hal/armv7a/pmap.c b/hal/armv7a/pmap.c index d72a5a26f..e974599ca 100644 --- a/hal/armv7a/pmap.c +++ b/hal/armv7a/pmap.c @@ -191,7 +191,7 @@ static void _pmap_asidDealloc(pmap_t *pmap) /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { pmap->pdir = vaddr; pmap->addr = p->addr; diff --git a/hal/armv7m/arch/pmap.h b/hal/armv7m/arch/pmap.h index 570a5a9f2..8372fc894 100644 --- a/hal/armv7m/arch/pmap.h +++ b/hal/armv7m/arch/pmap.h @@ -17,6 +17,7 @@ #define _PH_HAL_PMAP_ARMV7M_H_ #include "hal/types.h" +#include "syspage.h" /* Architecture dependent page attributes - used for mapping */ #define PGHD_PRESENT 0x01U @@ -55,7 +56,7 @@ typedef struct _page_t { typedef struct _pmap_t { void *start; void *end; - u32 regions; + const hal_syspage_part_t *hal; } pmap_t; #endif diff --git a/hal/armv7m/pmap.c b/hal/armv7m/pmap.c index 61bfb24a4..ae3bea2ce 100644 --- a/hal/armv7m/pmap.c +++ b/hal/armv7m/pmap.c @@ -17,9 +17,14 @@ #include "config.h" #include "syspage.h" #include "halsyspage.h" +#include "lib/lib.h" #include #include + +#define MPU_BASE ((volatile u32 *)0xe000ed90U) + + /* clang-format off */ enum { mpu_type, mpu_ctrl, mpu_rnr, mpu_rbar, mpu_rasr, mpu_rbar_a1, mpu_rasr_a1, mpu_rbar_a2, mpu_rasr_a2, mpu_rbar_a3, mpu_rasr_a3 }; @@ -36,15 +41,20 @@ extern void *_init_vectors; static struct { volatile u32 *mpu; - unsigned int kernelCodeRegion; spinlock_t lock; + unsigned int lastMPUCount; } pmap_common; /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { - pmap->regions = pmap_common.kernelCodeRegion; + if (prog != NULL) { + pmap->hal = prog->hal; + } + else { + pmap->hal = NULL; + } return 0; } @@ -55,55 +65,42 @@ addr_t pmap_destroy(pmap_t *pmap, unsigned int *i) } -static unsigned int pmap_map2region(unsigned int map) +/* parasoft-suppress-next-line MISRAC2012-DIR_4_3-a "Optimized context switching code" */ +void pmap_switch(pmap_t *pmap) { + const volatile u32 *RBAR_ADDR = MPU_BASE + mpu_rbar; + unsigned int allocCnt; + spinlock_ctx_t sc; unsigned int i; - unsigned int mask = 0; - - for (i = 0; i < sizeof(syspage->hs.mpu.map) / sizeof(*syspage->hs.mpu.map); ++i) { - if (map == syspage->hs.mpu.map[i]) { - mask |= (1UL << i); - } - } - - return mask; -} - + const u32 *tableCurrent; -int pmap_addMap(pmap_t *pmap, unsigned int map) -{ - unsigned int rmask = pmap_map2region(map); - if (rmask == 0U) { - return -1; - } + if ((pmap != NULL) && (pmap->hal != NULL)) { + hal_spinlockSet(&pmap_common.lock, &sc); - pmap->regions |= rmask; + allocCnt = pmap->hal->mpu.allocCnt; + tableCurrent = &pmap->hal->mpu.table[0].rbar; - return 0; -} + /* Disable MPU */ + hal_cpuDataMemoryBarrier(); + *(pmap_common.mpu + mpu_ctrl) &= ~1U; + + for (i = 0; i < max(allocCnt, pmap_common.lastMPUCount); i += 4U) { + /* region number update is done by writes to RBAR */ + __asm__ volatile( + "ldmia %[tableCurrent]!, {r3-r8, r10, r11} \n\t" /* Load 4 regions (rbar/rasr pairs) from table, update table pointer */ + "stmia %[mpu_rbar], {r3-r8, r10, r11} \n\t" /* Write 4 regions via RBAR/RASR and aliases */ + : [tableCurrent] "+&r"(tableCurrent) + : [mpu_rbar] "r"(RBAR_ADDR) + : "r3", "r4", "r5", "r6", "r7", "r8", "r10", "r11"); + } + /* Enable MPU */ + *(pmap_common.mpu + mpu_ctrl) |= 1U; + hal_cpuDataSyncBarrier(); + hal_cpuInstrBarrier(); -void pmap_switch(pmap_t *pmap) -{ - unsigned int i, cnt = syspage->hs.mpu.allocCnt; - spinlock_ctx_t sc; + pmap_common.lastMPUCount = allocCnt; - if (pmap != NULL) { - hal_spinlockSet(&pmap_common.lock, &sc); - for (i = 0; i < cnt; ++i) { - /* Select region */ - *(pmap_common.mpu + mpu_rnr) = i; - hal_cpuDataMemoryBarrier(); - - /* Enable/disable region according to the mask */ - if ((pmap->regions & (1UL << i)) != 0U) { - *(pmap_common.mpu + mpu_rasr) |= 1U; - } - else { - *(pmap_common.mpu + mpu_rasr) &= ~1U; - } - hal_cpuDataMemoryBarrier(); - } hal_spinlockClear(&pmap_common.lock, &sc); } } @@ -129,14 +126,23 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr) int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size) { + unsigned int i; const syspage_map_t *map = syspage_mapAddrResolve((addr_t)vaddr); - unsigned int rmask; if (map == NULL) { return 0; } - rmask = pmap_map2region(map->id); - return ((pmap->regions & rmask) == 0U) ? 0 : 1; + if (pmap->hal == NULL) { + /* Kernel pmap has access to everything */ + return 1; + } + + for (i = 0; i < pmap->hal->mpu.allocCnt; ++i) { + if (pmap->hal->mpu.map[i] == map->id) { + return 1; + } + } + return 0; } @@ -175,11 +181,8 @@ int pmap_segment(unsigned int i, void **vaddr, size_t *size, vm_prot_t *prot, vo /* parasoft-suppress-next-line MISRAC2012-DIR_4_3 "Assembly is required for low-level operations" */ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) { - const syspage_map_t *ikmap; - unsigned int ikregion; - u32 t; - addr_t pc; - unsigned int i, cnt = syspage->hs.mpu.allocCnt; + const unsigned int cnt = (syspage->hs.mpuType >> 8U) & 0xffU; + unsigned int i; (*vstart) = (void *)(((ptr_t)_init_vectors + 7U) & ~7U); (*vend) = (*((char **)vstart)) + SIZE_PAGE; @@ -189,8 +192,11 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) /* Initial size of kernel map */ pmap->end = (void *)((addr_t)&__bss_start + 32U * 1024U); + pmap->hal = NULL; + pmap_common.lastMPUCount = min(cnt, MPU_MAX_REGIONS); + /* Configure MPU */ - pmap_common.mpu = (void *)0xe000ed90U; + pmap_common.mpu = MPU_BASE; /* Disable MPU just in case */ *(pmap_common.mpu + mpu_ctrl) &= ~1U; @@ -201,17 +207,11 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) hal_cpuDataMemoryBarrier(); for (i = 0; i < cnt; ++i) { - t = syspage->hs.mpu.table[i].rbar; - if ((t & (1UL << 4)) == 0U) { - continue; - } - - *(pmap_common.mpu + mpu_rbar) = t; - hal_cpuDataMemoryBarrier(); + /* Select region */ + *(pmap_common.mpu + mpu_rnr) = i; - /* Disable regions for now */ - t = syspage->hs.mpu.table[i].rasr & ~1U; - *(pmap_common.mpu + mpu_rasr) = t; + /* Disable all regions for now */ + *(pmap_common.mpu + mpu_rasr) = 0U; hal_cpuDataMemoryBarrier(); } @@ -219,34 +219,5 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) *(pmap_common.mpu + mpu_ctrl) |= 1U; hal_cpuDataMemoryBarrier(); - /* FIXME HACK - * allow all programs to execute (and read) kernel code map. - * Needed because of hal_jmp, syscalls handler and signals handler. - * In these functions we need to switch to the user mode when still - * executing kernel code. This will cause memory management fault - * if the application does not have access to the kernel instruction - * map. Possible fix - place return to the user code in the separate - * region and allow this region instead. */ - - /* Find kernel code region */ - __asm__ volatile("\tmov %0, pc;" : "=r"(pc)); - ikmap = syspage_mapAddrResolve(pc); - if (ikmap == NULL) { - hal_consolePrint(ATTR_BOLD, "pmap: Kernel code map not found. Bad system config\n"); - for (;;) { - hal_cpuHalt(); - } - } - - ikregion = pmap_map2region(ikmap->id); - if (ikregion == 0U) { - hal_consolePrint(ATTR_BOLD, "pmap: Kernel code map has no assigned region. Bad system config\n"); - for (;;) { - hal_cpuHalt(); - } - } - - pmap_common.kernelCodeRegion = ikregion; - hal_spinlockCreate(&pmap_common.lock, "pmap"); } diff --git a/hal/armv7r/arch/pmap.h b/hal/armv7r/arch/pmap.h index 7704082cd..972aba55d 100644 --- a/hal/armv7r/arch/pmap.h +++ b/hal/armv7r/arch/pmap.h @@ -17,6 +17,7 @@ #define _PH_HAL_PMAP_ARMV7R_H_ #include "hal/types.h" +#include "syspage.h" #define PGHD_PRESENT 0x01U #define PGHD_USER 0x04U @@ -54,7 +55,7 @@ typedef struct _page_t { typedef struct _pmap_t { void *start; void *end; - u32 regions; + const hal_syspage_part_t *hal; } pmap_t; #endif diff --git a/hal/armv7r/pmap.c b/hal/armv7r/pmap.c index f3f9e3ab4..38bf108a3 100644 --- a/hal/armv7r/pmap.c +++ b/hal/armv7r/pmap.c @@ -18,6 +18,7 @@ #include "config.h" #include "syspage.h" #include "halsyspage.h" +#include "lib/lib.h" #include #include @@ -34,7 +35,9 @@ u8 _init_stack[NUM_CPUS][SIZE_INITIAL_KSTACK] __attribute__((aligned(8))); static struct { spinlock_t lock; - int mpu_enabled; + int mpuEnabled; + unsigned int lastMPUCount[NUM_CPUS]; + const hal_syspage_part_t *lastMPUConf[NUM_CPUS]; } pmap_common; @@ -103,9 +106,14 @@ static void pmap_mpu_disable(void) /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { - pmap->regions = 0U; + if (prog != NULL) { + pmap->hal = prog->hal; + } + else { + pmap->hal = NULL; + } return 0; } @@ -116,60 +124,47 @@ addr_t pmap_destroy(pmap_t *pmap, unsigned int *i) } -static unsigned int pmap_map2region(unsigned int map) +void pmap_switch(pmap_t *pmap) { - if (pmap_common.mpu_enabled == 0) { - return 1; - } - + const hal_syspage_part_t *hal; + unsigned int allocCnt; + spinlock_ctx_t sc; unsigned int i; - unsigned int mask = 0U; - - for (i = 0U; i < sizeof(syspage->hs.mpu.map) / sizeof(*syspage->hs.mpu.map); ++i) { - if (map == syspage->hs.mpu.map[i]) { - mask |= (1UL << i); - } - } - - return mask; -} - - -int pmap_addMap(pmap_t *pmap, unsigned int map) -{ - if (pmap_common.mpu_enabled == 0) { - return 0; - } - unsigned int rmask = pmap_map2region(map); - if (rmask == 0U) { - return -1; + if (pmap_common.mpuEnabled == 0) { + return; } - pmap->regions |= rmask; - - return 0; -} + if (pmap != NULL && pmap->hal != NULL) { + if (pmap->hal == pmap_common.lastMPUConf[hal_cpuGetID()]) { + return; + } + hal_spinlockSet(&pmap_common.lock, &sc); + hal = pmap->hal; + allocCnt = hal->mpu.allocCnt; -void pmap_switch(pmap_t *pmap) -{ - unsigned int i, cnt = syspage->hs.mpu.allocCnt; - spinlock_ctx_t sc; - if (pmap_common.mpu_enabled == 0) { - return; - } + /* Disable MPU */ + pmap_mpu_disable(); - if (pmap != NULL) { - hal_spinlockSet(&pmap_common.lock, &sc); - for (i = 0; i < cnt; ++i) { - /* Select region */ + for (i = 0; i < allocCnt; ++i) { pmap_mpu_setMemRegionNumber(i); + pmap_mpu_setMemRegionRbar(hal->mpu.table[i].rbar); + pmap_mpu_setMemRegionRasr(hal->mpu.table[i].rasr); + } - /* Enable/disable region according to the mask */ - pmap_mpu_setMemRegionStatus(((pmap->regions & (1UL << i)) != 0U) ? 1 : 0); + /* Disable all remaining regions */ + for (; i < pmap_common.lastMPUCount[hal_cpuGetID()]; i++) { + pmap_mpu_setMemRegionNumber(i); + pmap_mpu_setMemRegionStatus(0); } + /* Enable MPU */ + pmap_mpu_enable(); + + pmap_common.lastMPUCount[hal_cpuGetID()] = allocCnt; + pmap_common.lastMPUConf[hal_cpuGetID()] = hal; + hal_spinlockClear(&pmap_common.lock, &sc); } } @@ -195,9 +190,10 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr) int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size) { + unsigned int i; const syspage_map_t *map; - unsigned int rmask; - if (pmap_common.mpu_enabled == 0) { + + if (pmap_common.mpuEnabled == 0) { return 1; } @@ -205,9 +201,18 @@ int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size) if (map == NULL) { return 0; } - rmask = pmap_map2region(map->id); - return ((pmap->regions & rmask) == 0U) ? 0 : 1; + if (pmap->hal == NULL) { + /* Kernel pmap has access to everything */ + return 1; + } + + for (i = 0; i < pmap->hal->mpu.allocCnt; ++i) { + if (pmap->hal->mpu.map[i] == map->id) { + return 1; + } + } + return 0; } @@ -245,10 +250,8 @@ int pmap_segment(unsigned int i, void **vaddr, size_t *size, vm_prot_t *prot, vo void _pmap_init(pmap_t *pmap, void **vstart, void **vend) { - u32 t; + const unsigned int cnt = (syspage->hs.mpuType >> 8U) & 0xffU; unsigned int i; - unsigned int cnt = syspage->hs.mpu.allocCnt; - *vstart = (void *)(((ptr_t)&_end + 7U) & ~7U); *vend = (*((char **)vstart)) + SIZE_PAGE; @@ -258,28 +261,26 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) pmap->end = (void *)((addr_t)&__bss_start + 32U * 1024U); - pmap->regions = (1UL << cnt) - 1U; + pmap->hal = NULL; + for (i = 0; i < (unsigned int)NUM_CPUS; i++) { + pmap_common.lastMPUCount[i] = min(cnt, MPU_MAX_REGIONS); + pmap_common.lastMPUConf[i] = NULL; + } if (cnt == 0U) { hal_spinlockCreate(&pmap_common.lock, "pmap"); - pmap_common.mpu_enabled = 0; + pmap_common.mpuEnabled = 0; return; } - pmap_common.mpu_enabled = 1; + pmap_common.mpuEnabled = 1; /* Disable MPU that may have been enabled before */ pmap_mpu_disable(); for (i = 0; i < cnt; ++i) { pmap_mpu_setMemRegionNumber(i); - t = syspage->hs.mpu.table[i].rbar; - if ((t & (0x1U << 4)) == 0U) { - continue; - } - - pmap_mpu_setMemRegionRbar(t); - pmap_mpu_setMemRegionRasr(syspage->hs.mpu.table[i].rasr); /* Enable all regions */ + pmap_mpu_setMemRegionStatus(0); } /* Enable MPU */ diff --git a/hal/armv8m/arch/pmap.h b/hal/armv8m/arch/pmap.h index f68142162..f4253a1f7 100644 --- a/hal/armv8m/arch/pmap.h +++ b/hal/armv8m/arch/pmap.h @@ -17,6 +17,7 @@ #define _PH_HAL_PMAP_ARMV8M_H_ #include "hal/types.h" +#include "syspage.h" #define PGHD_PRESENT 0x01U #define PGHD_USER 0x04U @@ -54,7 +55,7 @@ typedef struct _page_t { typedef struct _pmap_t { void *start; void *end; - u32 regions; + const hal_syspage_part_t *hal; } pmap_t; #endif diff --git a/hal/armv8m/mcx/n94x/config.h b/hal/armv8m/mcx/n94x/config.h index 135448889..8cf5a46de 100644 --- a/hal/armv8m/mcx/n94x/config.h +++ b/hal/armv8m/mcx/n94x/config.h @@ -22,7 +22,9 @@ #ifndef __ASSEMBLY__ +#include "hal/types.h" #include "include/arch/armv8m/mcx/syspage.h" +#include "include/syspage.h" #include "mcxn94x.h" #define HAL_NAME_PLATFORM "MCX N94x " diff --git a/hal/armv8m/pmap.c b/hal/armv8m/pmap.c index 4fbd8ef37..ed093a5e6 100644 --- a/hal/armv8m/pmap.c +++ b/hal/armv8m/pmap.c @@ -17,6 +17,7 @@ #include "config.h" #include "syspage.h" #include "halsyspage.h" +#include "lib/lib.h" #include #include @@ -43,16 +44,21 @@ extern void *_init_vectors; static struct { volatile u32 *mpu; - unsigned int kernelCodeRegion; spinlock_t lock; - int mpu_enabled; + int mpuEnabled; + unsigned int lastMPUCount; } pmap_common; /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { - pmap->regions = pmap_common.kernelCodeRegion; + if (prog != NULL) { + pmap->hal = prog->hal; + } + else { + pmap->hal = NULL; + } return 0; } @@ -63,67 +69,46 @@ addr_t pmap_destroy(pmap_t *pmap, unsigned int *i) } -static unsigned int pmap_map2region(unsigned int map) +/* parasoft-suppress-next-line MISRAC2012-DIR_4_3-a "Optimized context switching code" */ +void pmap_switch(pmap_t *pmap) { + const volatile u32 *RBAR_ADDR = MPU_BASE + mpu_rbar; + unsigned int allocCnt; + spinlock_ctx_t sc; unsigned int i; - unsigned int mask = 0; - - if (pmap_common.mpu_enabled == 0) { - return 1; - } + const u32 *tableCurrent; - for (i = 0; i < sizeof(syspage->hs.mpu.map) / sizeof(*syspage->hs.mpu.map); ++i) { - if (map == syspage->hs.mpu.map[i]) { - mask |= (1UL << i); - } + if (pmap_common.mpuEnabled == 0) { + return; } - return mask; -} + if (pmap != NULL && pmap->hal != NULL) { + hal_spinlockSet(&pmap_common.lock, &sc); + allocCnt = pmap->hal->mpu.allocCnt; + tableCurrent = &pmap->hal->mpu.table[0].rbar; -int pmap_addMap(pmap_t *pmap, unsigned int map) -{ - unsigned int rmask; - if (pmap_common.mpu_enabled == 0) { - return 0; - } - - rmask = pmap_map2region(map); - if (rmask == 0U) { - return -1; - } + /* Disable MPU */ + hal_cpuDataMemoryBarrier(); + *(pmap_common.mpu + mpu_ctrl) &= ~1U; - pmap->regions |= rmask; + for (i = 0; i < max(allocCnt, pmap_common.lastMPUCount); i += 4U) { + *(pmap_common.mpu + mpu_rnr) = i; + __asm__ volatile( + "ldmia %[tableCurrent]!, {r3-r8, r10, r11} \n\t" /* Load 4 regions (rbar/rlar pairs) from table, update table pointer */ + "stmia %[mpu_rbar], {r3-r8, r10, r11} \n\t" /* Write 4 regions via RBAR/RLAR and aliases */ + : [tableCurrent] "+&r"(tableCurrent) + : [mpu_rbar] "r"(RBAR_ADDR) + : "r3", "r4", "r5", "r6", "r7", "r8", "r10", "r11"); + } - return 0; -} + /* Enable MPU */ + *(pmap_common.mpu + mpu_ctrl) |= 1U; + hal_cpuDataSyncBarrier(); + hal_cpuInstrBarrier(); + pmap_common.lastMPUCount = allocCnt; -void pmap_switch(pmap_t *pmap) -{ - unsigned int i, cnt = syspage->hs.mpu.allocCnt; - spinlock_ctx_t sc; - if (pmap_common.mpu_enabled == 0) { - return; - } - - if (pmap != NULL) { - hal_spinlockSet(&pmap_common.lock, &sc); - for (i = 0; i < cnt; ++i) { - /* Select region */ - *(pmap_common.mpu + mpu_rnr) = i; - hal_cpuDataMemoryBarrier(); - - /* Enable/disable region according to the mask */ - if ((pmap->regions & (1UL << i)) != 0UL) { - *(pmap_common.mpu + mpu_rlar) |= 1U; - } - else { - *(pmap_common.mpu + mpu_rlar) &= ~1U; - } - hal_cpuDataMemoryBarrier(); - } hal_spinlockClear(&pmap_common.lock, &sc); } } @@ -149,8 +134,8 @@ addr_t pmap_resolve(pmap_t *pmap, void *vaddr) int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size) { + unsigned int i; const syspage_map_t *map = syspage_mapAddrResolve((addr_t)vaddr); - unsigned int rmask; addr_t addr_end = (addr_t)vaddr + size; /* Check for potential arithmetic overflow. `addr_end` is allowed to be 0, * as it represents the top of memory. */ @@ -158,13 +143,21 @@ int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size) return 0; } - if (pmap_common.mpu_enabled == 0) { + if (pmap_common.mpuEnabled == 0) { return 1; } - rmask = pmap_map2region(map->id); + if (pmap->hal == NULL) { + /* Kernel pmap has access to everything */ + return 1; + } - return ((pmap->regions & rmask) != 0U) ? 1 : 0; + for (i = 0; i < pmap->hal->mpu.allocCnt; ++i) { + if (pmap->hal->mpu.map[i] == map->id) { + return 1; + } + } + return 0; } @@ -202,9 +195,8 @@ int pmap_segment(unsigned int i, void **vaddr, size_t *size, vm_prot_t *prot, vo void _pmap_init(pmap_t *pmap, void **vstart, void **vend) { - const syspage_map_t *ikmap; - unsigned int ikregion; - unsigned int i, cnt = syspage->hs.mpu.allocCnt; + const unsigned int cnt = (syspage->hs.mpuType >> 8U) & 0xffU; + unsigned int i; (*vstart) = (void *)(((ptr_t)_init_vectors + 7U) & ~7U); (*vend) = (*((char **)vstart)) + SIZE_PAGE; @@ -214,20 +206,19 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) /* Initial size of kernel map */ pmap->end = (void *)((addr_t)&__bss_start + 32U * 1024U); - /* Enable all regions for kernel */ - pmap->regions = (1UL << cnt) - 1UL; + pmap->hal = NULL; + pmap_common.lastMPUCount = min(cnt, MPU_MAX_REGIONS); /* Configure MPU */ pmap_common.mpu = MPU_BASE; hal_spinlockCreate(&pmap_common.lock, "pmap"); if (cnt == 0U) { - pmap_common.mpu_enabled = 0; - pmap_common.kernelCodeRegion = 0; + pmap_common.mpuEnabled = 0; return; } - pmap_common.mpu_enabled = 1; + pmap_common.mpuEnabled = 1; /* Disable MPU just in case */ *(pmap_common.mpu + mpu_ctrl) &= ~1U; @@ -241,42 +232,12 @@ void _pmap_init(pmap_t *pmap, void **vstart, void **vend) for (i = 0; i < cnt; ++i) { /* Select MPU region to configure */ *(pmap_common.mpu + mpu_rnr) = i; - hal_cpuDataMemoryBarrier(); - - *(pmap_common.mpu + mpu_rbar) = syspage->hs.mpu.table[i].rbar; - hal_cpuDataMemoryBarrier(); - /* Disable regions for now */ - *(pmap_common.mpu + mpu_rlar) = syspage->hs.mpu.table[i].rlar & ~1U; - hal_cpuDataMemoryBarrier(); + /* Disable all regions for now */ + *(pmap_common.mpu + mpu_rlar) = 0U; } /* Enable MPU */ *(pmap_common.mpu + mpu_ctrl) |= 1U; - hal_cpuDataMemoryBarrier(); - - /* FIXME HACK - * allow all programs to execute (and read) kernel code map. - * Needed because of hal_jmp, syscalls handler and signals handler. - * In these functions we need to switch to the user mode when still - * executing kernel code. This will cause memory management fault - * if the application does not have access to the kernel instruction - * map. Possible fix - place return to the user code in the separate - * region and allow this region instead. */ - - /* Find kernel code region */ - /* parasoft-suppress-next-line MISRAC2012-RULE_11_1 "We need address of this function in numeric type" */ - ikmap = syspage_mapAddrResolve((addr_t)_pmap_init); - if (ikmap != NULL) { - ikregion = pmap_map2region(ikmap->id); - } - - if ((ikmap == NULL) || (ikregion == 0U)) { - hal_consolePrint(ATTR_BOLD, "pmap: Kernel code map not found or has no regions. Bad system config\n"); - for (;;) { - hal_cpuHalt(); - } - } - - pmap_common.kernelCodeRegion = ikregion; + hal_cpuDataSyncBarrier(); } diff --git a/hal/armv8r/pmap.c b/hal/armv8r/pmap.c index 00677aad0..e5aaff90d 100644 --- a/hal/armv8r/pmap.c +++ b/hal/armv8r/pmap.c @@ -29,7 +29,7 @@ u8 _init_stack[NUM_CPUS][SIZE_INITIAL_KSTACK] __attribute__((aligned(8))); /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { return 0; } diff --git a/hal/ia32/pmap.c b/hal/ia32/pmap.c index e44342f46..6e4744b01 100644 --- a/hal/ia32/pmap.c +++ b/hal/ia32/pmap.c @@ -39,7 +39,7 @@ static struct { /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { u32 i, pages; pmap->pdir = vaddr; diff --git a/hal/pmap.h b/hal/pmap.h index e2968afb3..6885928d4 100644 --- a/hal/pmap.h +++ b/hal/pmap.h @@ -20,6 +20,7 @@ #include "lib/attrs.h" #include +#include "syspage.h" #ifndef NOMMU @@ -31,14 +32,12 @@ MAYBE_UNUSED static inline int pmap_belongs(pmap_t *pmap, void *addr) #else -int pmap_addMap(pmap_t *pmap, unsigned int map); - int pmap_isAllowed(pmap_t *pmap, const void *vaddr, size_t size); #endif -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr); +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr); addr_t pmap_destroy(pmap_t *pmap, unsigned int *i); diff --git a/hal/riscv64/_init.S b/hal/riscv64/_init.S index a00a326ef..ec3c88699 100644 --- a/hal/riscv64/_init.S +++ b/hal/riscv64/_init.S @@ -176,5 +176,5 @@ _init_core: .section ".bss" .align 3 _hal_syspageCopied: - .zero 0x600 + .zero 0x700 .size _hal_syspageCopied, .-_hal_syspageCopied diff --git a/hal/riscv64/pmap.c b/hal/riscv64/pmap.c index 305e5b241..1eda3e8a7 100644 --- a/hal/riscv64/pmap.c +++ b/hal/riscv64/pmap.c @@ -101,7 +101,7 @@ addr_t pmap_getKernelStart(void) /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { unsigned int i, pages; ptr_t va; diff --git a/hal/sparcv8leon/pmap-nommu.c b/hal/sparcv8leon/pmap-nommu.c index efbdce542..1014c00d8 100644 --- a/hal/sparcv8leon/pmap-nommu.c +++ b/hal/sparcv8leon/pmap-nommu.c @@ -28,7 +28,7 @@ extern void *_init_stack; /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { return 0; } @@ -40,12 +40,6 @@ addr_t pmap_destroy(pmap_t *pmap, unsigned int *i) } -int pmap_addMap(pmap_t *pmap, unsigned int map) -{ - return 0; -} - - void pmap_switch(pmap_t *pmap) { return; diff --git a/hal/sparcv8leon/pmap.c b/hal/sparcv8leon/pmap.c index 18fdf74b1..a2f1aab84 100644 --- a/hal/sparcv8leon/pmap.c +++ b/hal/sparcv8leon/pmap.c @@ -198,7 +198,7 @@ static void _pmap_contextDealloc(pmap_t *pmap) /* Function creates empty page table */ -int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, void *vaddr) +int pmap_create(pmap_t *pmap, pmap_t *kpmap, page_t *p, const syspage_prog_t *prog, void *vaddr) { pmap->pdir1 = vaddr; pmap->context = CONTEXT_INVALID; diff --git a/include/arch/aarch64/zynqmp/syspage.h b/include/arch/aarch64/zynqmp/syspage.h index 2abbc61be..2c52e6070 100644 --- a/include/arch/aarch64/zynqmp/syspage.h +++ b/include/arch/aarch64/zynqmp/syspage.h @@ -21,4 +21,8 @@ typedef struct { } __attribute__((packed)) hal_syspage_t; +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/arch/armv7a/imx6ull/syspage.h b/include/arch/armv7a/imx6ull/syspage.h index 1543fc8bc..0f859d848 100644 --- a/include/arch/armv7a/imx6ull/syspage.h +++ b/include/arch/armv7a/imx6ull/syspage.h @@ -21,4 +21,9 @@ typedef struct { int dummy; } __attribute__((packed)) hal_syspage_t; + +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/arch/armv7a/zynq7000/syspage.h b/include/arch/armv7a/zynq7000/syspage.h index e312c7184..125e7f80e 100644 --- a/include/arch/armv7a/zynq7000/syspage.h +++ b/include/arch/armv7a/zynq7000/syspage.h @@ -21,4 +21,8 @@ typedef struct { } __attribute__((packed)) hal_syspage_t; +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/arch/armv7m/imxrt/syspage.h b/include/arch/armv7m/imxrt/syspage.h index ae1dcee8f..cbf29b955 100644 --- a/include/arch/armv7m/imxrt/syspage.h +++ b/include/arch/armv7m/imxrt/syspage.h @@ -18,16 +18,25 @@ #define _PH_SYSPAGE_IMXRT_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rasr; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ - } __attribute__((packed)) mpu; + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; + } mpu; +} hal_syspage_part_t; + + +typedef struct { + unsigned int mpuType; unsigned int bootReason; } __attribute__((packed)) hal_syspage_t; diff --git a/include/arch/armv7m/stm32/syspage.h b/include/arch/armv7m/stm32/syspage.h index aaa8ef7cc..faca72721 100644 --- a/include/arch/armv7m/stm32/syspage.h +++ b/include/arch/armv7m/stm32/syspage.h @@ -18,16 +18,25 @@ #define _PH_SYSPAGE_STM32_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rasr; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ - } __attribute__((packed)) mpu; + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; + } mpu; +} hal_syspage_part_t; + + +typedef struct { + unsigned int mpuType; unsigned int bootReason; } __attribute__((packed)) hal_syspage_t; diff --git a/include/arch/armv7r/tda4vm/syspage.h b/include/arch/armv7r/tda4vm/syspage.h index eae556bb5..69855d460 100644 --- a/include/arch/armv7r/tda4vm/syspage.h +++ b/include/arch/armv7r/tda4vm/syspage.h @@ -17,17 +17,26 @@ #define _PH_SYSPAGE_ARMV7R_TDA4VM_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { - int resetReason; struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rasr; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; } __attribute__((packed)) mpu; +} __attribute__((packed)) hal_syspage_part_t; + + +typedef struct { + int resetReason; + unsigned int mpuType; } __attribute__((packed)) hal_syspage_t; diff --git a/include/arch/armv7r/zynqmp/syspage.h b/include/arch/armv7r/zynqmp/syspage.h index 31c94eecc..8dc5c4158 100644 --- a/include/arch/armv7r/zynqmp/syspage.h +++ b/include/arch/armv7r/zynqmp/syspage.h @@ -17,17 +17,26 @@ #define _PH_SYSPAGE_ARMV7R_ZYNQMP_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { - int resetReason; struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rasr; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; } __attribute__((packed)) mpu; +} __attribute__((packed)) hal_syspage_part_t; + + +typedef struct { + int resetReason; + unsigned int mpuType; } __attribute__((packed)) hal_syspage_t; diff --git a/include/arch/armv8m/mcx/syspage.h b/include/arch/armv8m/mcx/syspage.h index 2f6721505..607584b40 100644 --- a/include/arch/armv8m/mcx/syspage.h +++ b/include/arch/armv8m/mcx/syspage.h @@ -18,16 +18,25 @@ #define _PH_SYSPAGE_MCXN94X_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rlar; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ - } __attribute__((packed)) mpu; + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; + } mpu; +} hal_syspage_part_t; + + +typedef struct { + unsigned int mpuType; } __attribute__((packed)) hal_syspage_t; #endif diff --git a/include/arch/armv8m/nrf/syspage.h b/include/arch/armv8m/nrf/syspage.h index 6fc79e590..ba0315bc2 100644 --- a/include/arch/armv8m/nrf/syspage.h +++ b/include/arch/armv8m/nrf/syspage.h @@ -18,16 +18,25 @@ #define _PH_SYSPAGE_NRF91_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { struct { - unsigned int type; - unsigned int allocCnt; struct { unsigned int rbar; unsigned int rlar; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ - } __attribute__((packed)) mpu; + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; + } mpu; +} hal_syspage_part_t; + + +typedef struct { + unsigned int mpuType; } __attribute__((packed)) hal_syspage_t; #endif diff --git a/include/arch/armv8m/stm32/syspage.h b/include/arch/armv8m/stm32/syspage.h index 1f1e50c64..c0495f82a 100644 --- a/include/arch/armv8m/stm32/syspage.h +++ b/include/arch/armv8m/stm32/syspage.h @@ -18,17 +18,25 @@ #define _PH_SYSPAGE_STM32_H_ +#ifndef MPU_MAX_REGIONS +#define MPU_MAX_REGIONS 16U +#endif + + typedef struct { struct { - unsigned int type; - unsigned int allocCnt; - unsigned int mair[2]; struct { unsigned int rbar; unsigned int rlar; - } table[16] __attribute__((aligned(8))); - unsigned int map[16]; /* ((unsigned int)-1) = map is not assigned */ - } __attribute__((packed)) mpu; + } table[MPU_MAX_REGIONS] __attribute__((aligned(8))); + unsigned int map[MPU_MAX_REGIONS]; /* ((unsigned int)-1) = map is not assigned */ + unsigned int allocCnt; + } mpu; +} hal_syspage_part_t; + + +typedef struct { + unsigned int mpuType; unsigned int bootReason; } __attribute__((packed)) hal_syspage_t; diff --git a/include/arch/armv8r/mps3an536/syspage.h b/include/arch/armv8r/mps3an536/syspage.h index 1dce97769..52bfa3dde 100644 --- a/include/arch/armv8r/mps3an536/syspage.h +++ b/include/arch/armv8r/mps3an536/syspage.h @@ -22,4 +22,8 @@ typedef struct { } __attribute__((packed)) hal_syspage_t; +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/arch/ia32/syspage.h b/include/arch/ia32/syspage.h index e143489d9..5b4ece5ee 100644 --- a/include/arch/ia32/syspage.h +++ b/include/arch/ia32/syspage.h @@ -57,5 +57,8 @@ typedef struct { } __attribute__((packed)) graphmode; /* Graphics mode info */ } __attribute__((packed)) hal_syspage_t; +typedef struct { + int dummy; +} hal_syspage_part_t; #endif diff --git a/include/arch/riscv64/syspage.h b/include/arch/riscv64/syspage.h index 5044f1eef..5c4507c1d 100644 --- a/include/arch/riscv64/syspage.h +++ b/include/arch/riscv64/syspage.h @@ -21,4 +21,9 @@ typedef struct { unsigned int boothartId; } __attribute__((packed)) hal_syspage_t; + +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/arch/sparcv8leon/syspage.h b/include/arch/sparcv8leon/syspage.h index 020190ddb..deb716cf1 100644 --- a/include/arch/sparcv8leon/syspage.h +++ b/include/arch/sparcv8leon/syspage.h @@ -21,4 +21,9 @@ typedef struct { int dummy; } __attribute__((packed)) hal_syspage_t; + +typedef struct { + int dummy; +} hal_syspage_part_t; + #endif diff --git a/include/sysinfo.h b/include/sysinfo.h index 1c85108e8..3f6480daa 100644 --- a/include/sysinfo.h +++ b/include/sysinfo.h @@ -79,9 +79,14 @@ typedef struct { /* TODO: Consider changing type of kmaps mapsz from int to unsigned int */ typedef struct _meminfo_t { struct { - unsigned int alloc, free, boot, sz; + unsigned int alloc, total, boot, sz; int mapsz; - pageinfo_t *map; + int mapidx; + struct { + unsigned int alloc, free, boot; + int mapsz; + pageinfo_t *map; + } map; } page; struct { diff --git a/include/syspage.h b/include/syspage.h index d9ce36cd8..8bf8619a4 100644 --- a/include/syspage.h +++ b/include/syspage.h @@ -17,30 +17,72 @@ #define _PH_SYSPAGE_H_ +/* clang-format off */ enum { mAttrRead = 0x01, mAttrWrite = 0x02, mAttrExec = 0x04, mAttrShareable = 0x08, mAttrCacheable = 0x10, mAttrBufferable = 0x20 }; +enum { sFlagCommonCycle = 0x01 }; + + enum { console_default = 0, console_com0, console_com1, console_com2, console_com3, console_com4, console_com5, console_com6, console_com7, console_com8, console_com9, console_com10, console_com11, console_com12, console_com13, console_com14, console_com15, console_vga0 }; +/* clang-format on */ typedef struct _mapent_t { struct _mapent_t *next, *prev; + /* clang-format off */ enum { hal_entryReserved = 0, hal_entryTemp, hal_entryAllocated, hal_entryInvalid } type; + /* clang-format on */ addr_t start; addr_t end; } __attribute__((packed)) mapent_t; +typedef struct _syspage_sched_cycle_t { + unsigned char bgId; + size_t len; + + struct { + time_t stop; + unsigned char id; + } windows[]; +} __attribute__((packed)) syspage_sched_cycle_t; + + +typedef struct _syspage_sched_t { + size_t windowCnt; + size_t cycleCnt; + unsigned int flags; + + syspage_sched_cycle_t *cycles[]; +} __attribute__((packed)) syspage_sched_t; + + +typedef struct _syspage_part_t { + struct _syspage_part_t *next, *prev; + + unsigned int id; + char *name; + + size_t availableMem; + unsigned char schedWindow; + + hal_syspage_part_t *hal; +} __attribute__((packed)) syspage_part_t; + + typedef struct _syspage_prog_t { struct _syspage_prog_t *next, *prev; addr_t start; addr_t end; + syspage_part_t *partition; + char *argv; size_t imapSz; @@ -48,6 +90,8 @@ typedef struct _syspage_prog_t { size_t dmapSz; unsigned char *dmaps; + + hal_syspage_part_t *hal; } __attribute__((packed)) syspage_prog_t; @@ -72,8 +116,10 @@ typedef struct { addr_t pkernel; /* Physical address of kernel's beginning */ - syspage_map_t *maps; /* Maps list */ - syspage_prog_t *progs; /* Programs list*/ + syspage_map_t *maps; /* Maps list */ + syspage_sched_t *sched; /* Scheduler configuration */ + syspage_part_t *partitions; /* Partitions list */ + syspage_prog_t *progs; /* Programs list */ unsigned int console; /* Console ID defines in hal */ } __attribute__((packed)) syspage_t; diff --git a/main.c b/main.c index 84dace1b8..2c664cb08 100644 --- a/main.c +++ b/main.c @@ -88,7 +88,7 @@ static void main_initthr(void *unused) } argv[argc] = NULL; - res = proc_syspageSpawn(prog, vm_getSharedMap((int)prog->imaps[0]), vm_getSharedMap((int)prog->dmaps[0]), argv[0], argv); + res = proc_syspageSpawn(prog, vm_getPhysicalMap((int)prog->imaps[0]), vm_createPhMapList(prog->dmaps, prog->dmapSz), argv[0], argv); if (res < 0) { lib_printf("main: failed to spawn %s (%d)\n", argv[0], res); } @@ -131,7 +131,7 @@ int main(void) test_proc_exit(); #endif - (void)proc_start(main_initthr, NULL, (const char *)"init"); + (void)proc_start(main_initthr, NULL, (const char *)"init", NULL); /* Start scheduling, leave current stack */ hal_cpuEnableInterrupts(); diff --git a/perf/buffer-mem.c b/perf/buffer-mem.c index ee2871c9e..ffb1a7291 100644 --- a/perf/buffer-mem.c +++ b/perf/buffer-mem.c @@ -57,7 +57,7 @@ static void _bufferFree(void *data, page_t **pages) while (p != NULL) { *pages = p->next; - vm_pageFree(p); + vm_pageFree(p, NULL); sz += SIZE_PAGE; p = *pages; } @@ -80,7 +80,7 @@ static void *_bufferAlloc(page_t **pages, size_t sz) } for (v = data; (ptr_t)v < (ptr_t)data + sz; v += SIZE_PAGE) { - p = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); + p = vm_pageAlloc(buffer_common.kmap->phMaps, SIZE_PAGE, PAGE_OWNER_APP, NULL); if (p == NULL) { err = -ENOMEM; diff --git a/proc/msg-nommu.c b/proc/msg-nommu.c index b08c754d9..f916d2902 100644 --- a/proc/msg-nommu.c +++ b/proc/msg-nommu.c @@ -29,6 +29,19 @@ static struct { } msg_common; +static int msg_isAllowed(process_t *proc, port_t *p) +{ + if ((proc == NULL) || (p->owner == NULL) || + (proc->partition == NULL) || (p->owner->partition == NULL)) { + return EOK; + } + if (p->owner->partition != proc->partition) { + return -EACCES; + } + return EOK; +} + + int proc_send(u32 port, msg_t *msg) { port_t *p; @@ -44,6 +57,11 @@ int proc_send(u32 port, msg_t *msg) } sender = proc_current(); + err = msg_isAllowed(sender->process, p); + if (err != EOK) { + port_put(p, 0); + return err; + } kmsg.msg = msg; kmsg.src = sender->process; @@ -131,6 +149,11 @@ int proc_recv(u32 port, msg_t *msg, msg_rid_t *rid) if (p == NULL) { return -EINVAL; } + err = msg_isAllowed(proc_current()->process, p); + if (err != EOK) { + port_put(p, 0); + return err; + } hal_spinlockSet(&p->spinlock, &sc); diff --git a/proc/msg.c b/proc/msg.c index 65a826d03..8462fe54a 100644 --- a/proc/msg.c +++ b/proc/msg.c @@ -47,6 +47,7 @@ static void *msg_map(int dir, kmsg_t *kmsg, void *data, size_t size, process_t * int err; vm_flags_t flags; addr_t bpa, pa, epa; + ph_map_t **phMaps = (from != NULL && from->mapp != NULL) ? from->mapp->phMaps : msg_common.kmap->phMaps; if ((size == 0U) || (data == NULL)) { return NULL; @@ -113,7 +114,7 @@ static void *msg_map(int dir, kmsg_t *kmsg, void *data, size_t size, process_t * ml->boffs = boffs; bpa = pmap_resolve(&srcmap->pmap, data) & ~(SIZE_PAGE - 1U); - nbp = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); + nbp = vm_pageAlloc(phMaps, SIZE_PAGE, PAGE_OWNER_APP, (from == NULL) ? NULL : from->partition); ml->bp = nbp; if (nbp == NULL) { return NULL; @@ -154,7 +155,7 @@ static void *msg_map(int dir, kmsg_t *kmsg, void *data, size_t size, process_t * epa = pmap_resolve(&srcmap->pmap, vaddr) & ~(SIZE_PAGE - 1U); if ((boffs == 0U) || (eoffs >= boffs)) { - nep = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); + nep = vm_pageAlloc(phMaps, SIZE_PAGE, PAGE_OWNER_APP, (from == NULL) ? NULL : from->partition); ml->ep = nep; if (nep == NULL) { return NULL; @@ -192,14 +193,14 @@ static void msg_release(kmsg_t *kmsg) vm_map_t *map; if (kmsg->i.bp != NULL) { - vm_pageFree(kmsg->i.bp); + vm_pageFree(kmsg->i.bp, (kmsg->src == NULL) ? NULL : kmsg->src->partition); (void)vm_munmap(msg_common.kmap, kmsg->i.bvaddr, SIZE_PAGE); kmsg->i.bp = NULL; } if (kmsg->i.eoffs != 0U) { if (kmsg->i.ep != NULL) { - vm_pageFree(kmsg->i.ep); + vm_pageFree(kmsg->i.ep, (kmsg->src == NULL) ? NULL : kmsg->src->partition); } (void)vm_munmap(msg_common.kmap, kmsg->i.evaddr, SIZE_PAGE); kmsg->i.eoffs = 0; @@ -220,14 +221,14 @@ static void msg_release(kmsg_t *kmsg) } if (kmsg->o.bp != NULL) { - vm_pageFree(kmsg->o.bp); + vm_pageFree(kmsg->o.bp, (kmsg->src == NULL) ? NULL : kmsg->src->partition); (void)vm_munmap(msg_common.kmap, kmsg->o.bvaddr, SIZE_PAGE); kmsg->o.bp = NULL; } if (kmsg->o.eoffs != 0U) { if (kmsg->o.ep != NULL) { - vm_pageFree(kmsg->o.ep); + vm_pageFree(kmsg->o.ep, (kmsg->src == NULL) ? NULL : kmsg->src->partition); } (void)vm_munmap(msg_common.kmap, kmsg->o.evaddr, SIZE_PAGE); kmsg->o.eoffs = 0; @@ -347,6 +348,19 @@ static int msg_opack(kmsg_t *kmsg) } +static int msg_isAllowed(process_t *proc, port_t *p) +{ + if ((proc == NULL) || (p->owner == NULL) || + (proc->partition == NULL) || (p->owner->partition == NULL)) { + return EOK; + } + if (p->owner->partition != proc->partition) { + return -EACCES; + } + return EOK; +} + + int proc_send(u32 port, msg_t *msg) { port_t *p; @@ -367,6 +381,11 @@ int proc_send(u32 port, msg_t *msg) } sender = proc_current(); + err = msg_isAllowed(sender->process, p); + if (err != EOK) { + port_put(p, 0); + return err; + } hal_memcpy(&kmsg.msg, msg, sizeof(msg_t)); kmsg.src = sender->process; @@ -451,6 +470,12 @@ int proc_recv(u32 port, msg_t *msg, msg_rid_t *rid) return -EINVAL; } + err = msg_isAllowed(proc_current()->process, p); + if (err != EOK) { + port_put(p, 0); + return err; + } + hal_spinlockSet(&p->spinlock, &sc); while ((p->kmessages == NULL) && (p->closed == 0) && (err != -EINTR)) { diff --git a/proc/process.c b/proc/process.c index d43861161..d95632111 100644 --- a/proc/process.c +++ b/proc/process.c @@ -19,6 +19,8 @@ #include "include/errno.h" #include "include/signal.h" #include "include/syscalls.h" +#include "include/syspage.h" +#include "syspage.h" #include "vm/vm.h" #include "lib/lib.h" #include "posix/posix.h" @@ -46,8 +48,8 @@ typedef struct _process_spawn_t { vm_object_t *object; off_t offset; size_t size; - vm_map_t *map; - vm_map_t *imap; + ph_map_t *imap; + ph_map_t **dmaps; const syspage_prog_t *prog; char **argv; @@ -63,6 +65,7 @@ static struct { lock_t lock; idtree_t id; int idcounter; + partition_t *partitions; } process_common; @@ -99,6 +102,7 @@ static void process_destroy(process_t *p) if (mapp != NULL) { vm_mapDestroy(p, mapp); + vm_kfree(mapp->phMaps); } if (imapp != NULL) { @@ -176,7 +180,7 @@ static int process_alloc(process_t *process) } -int proc_start(startFn_t start, void *arg, const char *path) +int proc_start(startFn_t start, void *arg, const char *path, partition_t *partition) { int err = EOK; process_t *process; @@ -205,6 +209,7 @@ int proc_start(startFn_t start, void *arg, const char *path) process->ghosts = NULL; process->reaper = NULL; process->refs = 1; + process->partition = partition; (void)proc_lockInit(&process->lock, &proc_lockAttrDefault, "process"); @@ -1088,7 +1093,6 @@ static void process_exec(thread_t *current, process_spawn_t *spawn) void *stack, *entry = NULL; int err = EOK, count; void *cleanupFn = NULL; - unsigned int i = 0; spinlock_ctx_t sc; const struct stackArg args[] = { { &spawn->envp, sizeof(spawn->envp) }, @@ -1101,33 +1105,17 @@ static void process_exec(thread_t *current, process_spawn_t *spawn) current->process->envp = spawn->envp; #ifndef NOMMU - err = vm_mapCreate(¤t->process->map, (void *)(VADDR_MIN + SIZE_PAGE), (void *)VADDR_USR_MAX); + err = vm_mapCreate(¤t->process->map, (void *)(VADDR_MIN + SIZE_PAGE), (void *)VADDR_USR_MAX, spawn->dmaps); if (err == EOK) { proc_changeMap(current->process, ¤t->process->map, NULL, ¤t->process->map.pmap); } - (void)i; #else - (void)pmap_create(¤t->process->map.pmap, NULL, NULL, NULL); - proc_changeMap(current->process, (spawn->map != NULL) ? spawn->map : process_common.kmap, spawn->imap, ¤t->process->map.pmap); - current->process->entries = NULL; - - if (spawn->prog != NULL) { - /* Add instruction maps */ - for (i = 0; i < spawn->prog->imapSz; ++i) { - if (err != EOK) { - break; - } - err = pmap_addMap(current->process->pmapp, spawn->prog->imaps[i]); - } - - /* Add data/io maps */ - for (i = 0; i < spawn->prog->dmapSz; ++i) { - if (err != EOK) { - break; - } - err = pmap_addMap(current->process->pmapp, spawn->prog->dmaps[i]); - } + err = vm_mapCreate(¤t->process->map, NULL, NULL, spawn->dmaps); + if (err == EOK) { + (void)pmap_create(¤t->process->map.pmap, NULL, NULL, spawn->prog, NULL); + proc_changeMap(current->process, ¤t->process->map, spawn->imap, ¤t->process->map.pmap); } + current->process->entries = NULL; #endif if (err == EOK) { @@ -1209,15 +1197,38 @@ static void proc_spawnThread(void *arg) } -static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, vm_map_t *imap, vm_map_t *map, off_t offset, size_t size, const char *path, char **argv, char **envp) +static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, ph_map_t *imap, ph_map_t **dmaps, off_t offset, size_t size, const char *path, char **argv, char **envp) { int pid; process_spawn_t spawn; spinlock_ctx_t sc; + partition_t *part; + process_t *proc = proc_current()->process; + + if (dmaps == NULL) { + return -EINVAL; + } + + if (prog != NULL) { + part = &process_common.partitions[prog->partition->id]; + } + else if ((proc != NULL) && (proc->partition != NULL)) { + part = proc->partition; + } + else { + part = NULL; + } + if ((proc != NULL) && + (proc->partition != NULL) && + (proc->partition != part)) { + vm_kfree(dmaps); + return -EACCES; + } if (argv != NULL) { argv = proc_copyargs(argv); if (argv == NULL) { + vm_kfree(dmaps); return -ENOMEM; } } @@ -1226,6 +1237,7 @@ static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, vm_map_t envp = proc_copyargs(envp); if (envp == NULL) { vm_kfree(argv); + vm_kfree(dmaps); return -ENOMEM; } } @@ -1238,13 +1250,13 @@ static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, vm_map_t spawn.argv = argv; spawn.envp = envp; spawn.parent = proc_current(); - spawn.map = map; spawn.imap = imap; + spawn.dmaps = dmaps; spawn.prog = prog; hal_spinlockCreate(&spawn.sl, "spawnsl"); - pid = proc_start(proc_spawnThread, &spawn, path); + pid = proc_start(proc_spawnThread, &spawn, path, part); if (pid > 0) { hal_spinlockSet(&spawn.sl, &sc); while (spawn.state == FORKING) { @@ -1255,6 +1267,7 @@ static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, vm_map_t else { vm_kfree(argv); vm_kfree(envp); + vm_kfree(dmaps); } hal_spinlockDestroy(&spawn.sl); @@ -1263,70 +1276,106 @@ static int proc_spawn(vm_object_t *object, const syspage_prog_t *prog, vm_map_t } +static ph_map_t **process_clonePhMapList(ph_map_t **phMaps) +{ + ph_map_t **cloned; + unsigned int phMapsz = 0; + + while (phMaps[phMapsz] != NULL) { + phMapsz++; + } + phMapsz++; + cloned = vm_kmalloc(sizeof(ph_map_t *) * phMapsz); + if (cloned == NULL) { + return NULL; + } + hal_memcpy(cloned, phMaps, sizeof(ph_map_t *) * phMapsz); + return cloned; +} + + int proc_fileSpawn(const char *path, char **argv, char **envp) { int err; oid_t oid; vm_object_t *object; + ph_map_t **phMaps; + process_t *process = proc_current()->process; err = proc_lookup(path, NULL, &oid); if (err < 0) { return err; } - err = vm_objectGet(&object, oid); + err = vm_objectGet(&object, oid, process->partition); if (err < 0) { return err; } - return proc_spawn(object, NULL, NULL, NULL, 0, object->size, path, argv, envp); + phMaps = process_clonePhMapList(process->mapp->phMaps); + if (phMaps == NULL) { + (void)vm_objectPut(object); + return -ENOMEM; + } + + return proc_spawn(object, NULL, NULL, phMaps, 0, object->size, path, argv, envp); } int proc_syspageSpawnName(const char *imap, const char *dmap, const char *name, char **argv) { - const syspage_map_t *sysMap, *codeMap; const syspage_prog_t *prog = syspage_progNameResolve(name); const unsigned int readExecAttr = (unsigned int)mAttrRead | (unsigned int)mAttrExec; const unsigned int readWriteAttr = (unsigned int)mAttrRead | (unsigned int)mAttrWrite; - vm_map_t *imapp = NULL, *dmapp; + const syspage_map_t *map; + ph_map_t *imapp = NULL, **dmaps; + unsigned int i; + int err; if (prog == NULL) { return -ENOENT; } if (imap == NULL) { - codeMap = syspage_mapIdResolve(prog->imaps[0]); + map = syspage_mapIdResolve(prog->imaps[0]); } else { - codeMap = syspage_mapNameResolve(imap); - if (codeMap == NULL) { - return -EINVAL; - } + map = syspage_mapNameResolve(imap); + } + if ((map == NULL) || ((map->attr & readExecAttr) != readExecAttr)) { + return -EINVAL; } + imapp = vm_getPhysicalMap((int)map->id); - if (codeMap != NULL) { - if ((codeMap->attr & readExecAttr) != readExecAttr) { + if (dmap == NULL) { + for (i = 0U; i < prog->dmapSz; i++) { + map = syspage_mapIdResolve(prog->dmaps[i]); + if ((map == NULL) || ((map->attr & readWriteAttr) != readWriteAttr)) { + return -EINVAL; + } + } + dmaps = vm_createPhMapList(prog->dmaps, prog->dmapSz); + } + else { + map = syspage_mapNameResolve(dmap); + if ((map == NULL) || ((map->attr & readWriteAttr) != readWriteAttr)) { return -EINVAL; } - - /* NOTE: imapp can be NULL */ - imapp = vm_getSharedMap((int)codeMap->id); + dmaps = vm_createPhMapList(&map->id, 1); } - - sysMap = (dmap == NULL) ? syspage_mapIdResolve(prog->dmaps[0]) : syspage_mapNameResolve(dmap); - if (sysMap == NULL || (sysMap->attr & readWriteAttr) != readWriteAttr) { - return -EINVAL; + if (dmaps == NULL) { + return -ENOMEM; } - dmapp = vm_getSharedMap((int)sysMap->id); - return proc_syspageSpawn(prog, imapp, dmapp, name, argv); + err = proc_syspageSpawn((const syspage_prog_t *)prog, imapp, dmaps, name, argv); + + return err; } -int proc_syspageSpawn(const syspage_prog_t *program, vm_map_t *imap, vm_map_t *map, const char *path, char **argv) +int proc_syspageSpawn(const syspage_prog_t *program, ph_map_t *imap, ph_map_t **dmaps, const char *path, char **argv) { - return proc_spawn(VM_OBJ_PHYSMEM, program, imap, map, (off_t)program->start, program->end - program->start, path, argv, NULL); + return proc_spawn(VM_OBJ_PHYSMEM, program, imap, dmaps, (off_t)program->start, program->end - program->start, path, argv, NULL); } @@ -1489,7 +1538,7 @@ int proc_vfork(void) spawn->parent = current; spawn->prog = NULL; - pid = proc_start(process_vforkThread, spawn, NULL); + pid = proc_start(process_vforkThread, spawn, NULL, (current->process != NULL) ? current->process->partition : NULL); if (pid < 0) { hal_spinlockDestroy(&spawn->sl); vm_kfree(spawn); @@ -1565,7 +1614,12 @@ static int process_copy(void) /* Avoid ustack access while map is invalid */ current->ustack = NULL; - if (vm_mapCreate(&process->map, parent->process->mapp->start, parent->process->mapp->stop) < 0) { + ph_map_t **phMaps = process_clonePhMapList(parent->process->mapp->phMaps); + if (phMaps == NULL) { + return -ENOMEM; + } + + if (vm_mapCreate(&process->map, parent->process->mapp->start, parent->process->mapp->stop, phMaps) < 0) { return -ENOMEM; } @@ -1683,6 +1737,11 @@ static int process_execve(thread_t *current) /* Restore kernel stack of parent thread */ if (parent != NULL) { process_restoreParentKstack(current, parent); + spawn->dmaps = process_clonePhMapList(parent->process->mapp->phMaps); + if (spawn->dmaps == NULL) { + current->process->exit = -ENOMEM; + proc_threadEnd(); + } } else { /* Reinitialize process */ @@ -1691,6 +1750,7 @@ static int process_execve(thread_t *current) proc_changeMap(current->process, NULL, NULL, NULL); pmap_switch(&process_common.kmap->pmap); + spawn->dmaps = map->phMaps; vm_mapDestroy(current->process, map); if (imap != NULL) { @@ -1762,7 +1822,7 @@ int proc_execve(const char *path, char **argv, char **envp) return err; } - err = vm_objectGet(&object, oid); + err = vm_objectGet(&object, oid, (current->process == NULL) ? NULL : current->process->partition); if (err < 0) { vm_kfree(kpath); vm_kfree(argv); @@ -1829,6 +1889,9 @@ int proc_sigpost(int pid, int sig) int _process_init(vm_map_t *kmap, vm_object_t *kernel) { + unsigned int cnt; + syspage_part_t *sysPart; + process_common.kmap = kmap; process_common.first = NULL; process_common.kernel = kernel; @@ -1836,6 +1899,21 @@ int _process_init(vm_map_t *kmap, vm_object_t *kernel) (void)proc_lockInit(&process_common.lock, &proc_lockAttrDefault, "process.common"); lib_idtreeInit(&process_common.id); + sysPart = syspage_partitionList(); + cnt = sysPart->prev->id + 1U; + process_common.partitions = vm_kmalloc(sizeof(partition_t) * cnt); + if (process_common.partitions == NULL) { + return -ENOMEM; + } + + do { + LIB_ASSERT_ALWAYS(sysPart->id < cnt, "Invalid partition ids in syspage", sysPart->id); + (void)proc_lockInit(&process_common.partitions[sysPart->id].lock, &proc_lockAttrDefault, "partition"); + process_common.partitions[sysPart->id].config = sysPart; + process_common.partitions[sysPart->id].usedMem = 0U; + sysPart = sysPart->next; + } while (sysPart != syspage_partitionList()); + (void)hal_exceptionsSetHandler(EXC_DEFAULT, process_exception); (void)hal_exceptionsSetHandler(EXC_UNDEFINED, process_illegal); return EOK; diff --git a/proc/process.h b/proc/process.h index 535fb0cf5..eee9965ad 100644 --- a/proc/process.h +++ b/proc/process.h @@ -30,6 +30,13 @@ typedef void (*sighandlerFn_t)(void); +typedef struct _partition_t { + lock_t lock; + size_t usedMem; + const syspage_part_t *config; +} partition_t; + + typedef struct _process_t { lock_t lock; @@ -48,6 +55,7 @@ typedef struct _process_t { vm_map_t *mapp; vm_map_t *imapp; pmap_t *pmapp; + partition_t *partition; int exit; unsigned int lazy : 1; @@ -93,7 +101,7 @@ void proc_kill(process_t *proc); void proc_reap(void); -int proc_start(startFn_t start, void *arg, const char *path); +int proc_start(startFn_t start, void *arg, const char *path, partition_t *partition); int proc_fileSpawn(const char *path, char **argv, char **envp); @@ -102,7 +110,7 @@ int proc_fileSpawn(const char *path, char **argv, char **envp); int proc_syspageSpawnName(const char *imap, const char *dmap, const char *name, char **argv); -int proc_syspageSpawn(const syspage_prog_t *program, vm_map_t *imap, vm_map_t *map, const char *path, char **argv); +int proc_syspageSpawn(const syspage_prog_t *program, ph_map_t *imap, ph_map_t **dmaps, const char *path, char **argv); int proc_execve(const char *path, char **argv, char **envp); diff --git a/proc/threads.c b/proc/threads.c index 8b54452b8..648112687 100644 --- a/proc/threads.c +++ b/proc/threads.c @@ -41,17 +41,31 @@ const struct lockAttr proc_lockAttrDefault = { .type = PH_LOCK_NORMAL }; /* Special empty queue value used to wakeup next enqueued thread. This is used to implement sticky conditions */ static thread_t *const wakeupPending = (void *)-1; + +#define NUM_PRIO 8U + +#define NO_WAKEUP ((time_t) - 1) + + +typedef struct { + thread_t *ready[NUM_PRIO]; + rbtree_t sleeping; + time_t sleepMin; /* minimum wakeup time from sleeping tree of this window */ + unsigned int sleepMinClaimed; /* CPU ID that claimed the task of waking up current sleepMin */ +} sched_window_t; + + static struct { vm_map_t *kmap; spinlock_t spinlock; lock_t lock; - thread_t *ready[8]; + + sched_window_t **windows; /* window 0 is for kernel threads and default partition */ + size_t *actWindow; /* Currently scheduled window for each CPU */ + syspage_sched_cycle_t **cycles; /* Scheduler cycle configuration for each CPU */ thread_t **current; time_t utcoffs; - /* Synchronized by spinlock */ - rbtree_t sleeping; - /* Synchronized by mutex */ unsigned int idcounter; idtree_t id; @@ -71,9 +85,7 @@ static struct { } threads_common; -_Static_assert(sizeof(threads_common.ready) / sizeof(threads_common.ready[0]) <= (u8)-1, "queue size must fit into priority type"); - -#define MAX_PRIO ((u8)(sizeof(threads_common.ready) / sizeof(threads_common.ready[0])) - 1U) +_Static_assert(NUM_PRIO <= (u8)-1, "queue size must fit into priority type"); static thread_t *_proc_current(void); @@ -167,61 +179,93 @@ static void _threads_waking(thread_t *t) /* - * Time management + * Scheduling windows helpers */ -static void _threads_updateWakeup(time_t now, thread_t *minimum) +static int threads_nonBackroundSchedWindows(void) { - thread_t *t; - time_t wakeup; + return (threads_common.cycles[hal_cpuGetID()]->len > 0U) ? 1 : 0; +} - if (minimum != NULL) { - t = minimum; + +static size_t proc_getSchedWindowId(const process_t *process) +{ + if ((process != NULL) && (process->partition != NULL)) { + return process->partition->config->schedWindow; } else { - t = lib_treeof(thread_t, sleeplinkage, lib_rbMinimum(threads_common.sleeping.root)); + /* Kernel threads are executing in special window 0 */ + return 0; } +} - if (t != NULL) { - if (now >= t->wakeup) { - wakeup = 1; + +static thread_t **proc_getReadyQueues(const process_t *process) +{ + return threads_common.windows[proc_getSchedWindowId(process)]->ready; +} + + +static time_t threads_schedWindowsCycleDuration(void) +{ + return threads_common.cycles[hal_cpuGetID()]->windows[threads_common.cycles[hal_cpuGetID()]->len - 1U].stop; +} + + +/* + * Time management + */ + + +static void _threads_updateWakeup(time_t now, thread_t *minimum, size_t windowId) +{ + if (minimum != NULL) { + if (now >= minimum->wakeup) { + threads_common.windows[windowId]->sleepMin = now; } else { - wakeup = t->wakeup - now; + threads_common.windows[windowId]->sleepMin = minimum->wakeup; + } + + if ((threads_common.windows[windowId]->sleepMin - now < SYSTICK_INTERVAL) && (threads_common.windows[windowId]->sleepMinClaimed == (unsigned int)-1)) { + hal_timerSetWakeup((unsigned int)threads_common.windows[windowId]->sleepMin); + threads_common.windows[windowId]->sleepMinClaimed = hal_cpuGetID(); } } else { - wakeup = SYSTICK_INTERVAL; + threads_common.windows[windowId]->sleepMin = NO_WAKEUP; } +} - if (wakeup > SYSTICK_INTERVAL + SYSTICK_INTERVAL / 8) { - wakeup = SYSTICK_INTERVAL; + +static void _threads_sleepingInsert(thread_t *t, time_t timeout) +{ + size_t windowId = proc_getSchedWindowId(t->process); + thread_t *minimum = lib_treeof(thread_t, sleeplinkage, lib_rbMinimum(threads_common.windows[windowId]->sleeping.root)); + + t->wakeup = timeout; + + if ((minimum == NULL) || (t->wakeup < minimum->wakeup)) { + minimum = t; + threads_common.windows[windowId]->sleepMinClaimed = (unsigned int)-1; } - hal_timerSetWakeup((unsigned int)wakeup); + (void)lib_rbInsert(&threads_common.windows[windowId]->sleeping, &t->sleeplinkage); + _threads_updateWakeup(_proc_gettimeRaw(), minimum, windowId); } -static int threads_timeintr(unsigned int n, cpu_context_t *context, void *arg) +static void _threads_dequeueAwakening(time_t now, size_t windowId) { thread_t *t; - time_t now; - spinlock_ctx_t sc; - /* parasoft-begin-suppress MISRAC2012-RULE_14_3 "hal_cpuGetID()'s return value might - * not be known at compile time for different architectures" */ - if (hal_cpuGetID() != 0U) { - /* Invoke scheduler */ - return 1; + if (threads_common.windows[windowId]->sleepMinClaimed != hal_cpuGetID()) { + return; } - /* parasoft-end-suppress MISRAC2012-RULE_14_3 */ - - hal_spinlockSet(&threads_common.spinlock, &sc); - now = _proc_gettimeRaw(); for (;;) { - t = lib_treeof(thread_t, sleeplinkage, lib_rbMinimum(threads_common.sleeping.root)); + t = lib_treeof(thread_t, sleeplinkage, lib_rbMinimum(threads_common.windows[windowId]->sleeping.root)); if (t == NULL || t->wakeup > now) { break; @@ -231,7 +275,24 @@ static int threads_timeintr(unsigned int n, cpu_context_t *context, void *arg) hal_cpuSetReturnValue(t->context, (void *)-ETIME); } - _threads_updateWakeup(now, t); + _threads_updateWakeup(now, t, windowId); + + threads_common.windows[windowId]->sleepMinClaimed = (unsigned int)-1; +} + + +static int threads_timeintr(unsigned int n, cpu_context_t *context, void *arg) +{ + time_t now; + spinlock_ctx_t sc; + unsigned int cpuId = hal_cpuGetID(); + + hal_spinlockSet(&threads_common.spinlock, &sc); + now = _proc_gettimeRaw(); + if (threads_nonBackroundSchedWindows() != 0) { + _threads_dequeueAwakening(now, threads_common.actWindow[cpuId]); + } + _threads_dequeueAwakening(now, threads_common.cycles[cpuId]->bgId); hal_spinlockClear(&threads_common.spinlock, &sc); @@ -340,17 +401,39 @@ __attribute__((noreturn)) void proc_longjmp(cpu_context_t *ctx) } +static time_t _threads_claimSleepingMin(time_t wakeup, time_t now, sched_window_t *window, unsigned int cpuId) +{ + if ((window->sleepMinClaimed != (unsigned int)-1) && (window->sleepMinClaimed != cpuId)) { + /* Already claimed by other CPU */ + return wakeup; + } + if ((window->sleepMin == NO_WAKEUP) || (window->sleepMin >= now + wakeup)) { + /* No thread to awaken before wakeup */ + return wakeup; + } + window->sleepMinClaimed = cpuId; + return window->sleepMin - now; +} + + static int _threads_checkSignal(thread_t *selected, process_t *proc, cpu_context_t *signalCtx, unsigned int oldmask, const int src); /* parasoft-suppress-next-line MISRAC2012-RULE_8_4 "Function is used externally within assembler code" */ int _threads_schedule(unsigned int n, cpu_context_t *context, void *arg) { - thread_t *current, *selected; + thread_t *current, *selected = NULL; unsigned int i; process_t *proc; cpu_context_t *signalCtx, *selCtx; unsigned int cpuId = hal_cpuGetID(); + sched_window_t *window = NULL; + thread_t **actReady = NULL; + thread_t **bgReady = NULL; + time_t wakeup = NO_WAKEUP; + time_t now = _proc_gettimeRaw(); + time_t cycleStartElapsed; + syspage_sched_cycle_t *cycle = threads_common.cycles[cpuId]; (void)arg; (void)n; @@ -358,6 +441,23 @@ int _threads_schedule(unsigned int n, cpu_context_t *context, void *arg) trace_eventSchedEnter(cpuId); + if (threads_nonBackroundSchedWindows() != 0) { + /* Update scheduling window to be scheduled in this call */ + cycleStartElapsed = now % threads_schedWindowsCycleDuration(); + i = 0; + while (cycleStartElapsed >= cycle->windows[i].stop) { + i++; + } + wakeup = cycle->windows[i].stop - cycleStartElapsed; + + i = cycle->windows[i].id; + threads_common.actWindow[cpuId] = i; + actReady = threads_common.windows[i]->ready; + window = threads_common.windows[i]; + } + + bgReady = threads_common.windows[cycle->bgId]->ready; + current = _proc_current(); threads_common.current[cpuId] = NULL; @@ -365,24 +465,34 @@ int _threads_schedule(unsigned int n, cpu_context_t *context, void *arg) if (current != NULL) { current->context = context; - /* Move thread to the end of queue */ + /* Move thread to the end of queue corresponding to its scheduling window */ if (current->state == READY) { - LIST_ADD(&threads_common.ready[current->priority], current); + LIST_ADD(&proc_getReadyQueues(current->process)[current->priority], current); _threads_preempted(current); } } /* Get next thread */ i = 0; - while (i < sizeof(threads_common.ready) / sizeof(thread_t *)) { - selected = threads_common.ready[i]; - if (selected == NULL) { + while (i < NUM_PRIO) { + if ((actReady != NULL) && (actReady[i] != NULL)) { + selected = actReady[i]; + LIST_REMOVE(&actReady[i], selected); + } + else if ((bgReady != NULL) && (bgReady[i] != NULL)) { + selected = bgReady[i]; + LIST_REMOVE(&bgReady[i], selected); + } + else if (threads_common.windows[0]->ready[i] != NULL) { + /* kernel threads window */ + selected = threads_common.windows[0]->ready[i]; + LIST_REMOVE(&threads_common.windows[0]->ready[i], selected); + } + else { i++; continue; } - LIST_REMOVE(&threads_common.ready[i], selected); - if (selected->exit == 0U) { break; } @@ -451,6 +561,22 @@ int _threads_schedule(unsigned int n, cpu_context_t *context, void *arg) /* Update CPU usage */ _threads_cpuTimeCalc(current, selected); + /* Set next wakeup before scheduling window finish if window other than background (0) exists */ + if ((wakeup == NO_WAKEUP) || (wakeup > SYSTICK_INTERVAL + SYSTICK_INTERVAL / 8)) { + wakeup = SYSTICK_INTERVAL; + } + /* Handle sleeping wakeups */ + if (window != NULL) { + wakeup = _threads_claimSleepingMin(wakeup, now, window, cpuId); + } + wakeup = _threads_claimSleepingMin(wakeup, now, threads_common.windows[cycle->bgId], cpuId); + + if (wakeup <= 0) { + wakeup = 1; + } + + hal_timerSetWakeup((unsigned int)wakeup); + trace_eventSchedExit(cpuId); return EOK; @@ -541,7 +667,7 @@ int proc_threadCreate(process_t *process, startFn_t start, int *id, u8 priority, spinlock_ctx_t sc; int err; - if (priority >= sizeof(threads_common.ready) / sizeof(thread_t *)) { + if (priority >= NUM_PRIO) { return -EINVAL; } @@ -626,7 +752,7 @@ int proc_threadCreate(process_t *process, startFn_t start, int *id, u8 priority, /* Insert thread to scheduler queue */ _threads_waking(t); - LIST_ADD(&threads_common.ready[priority], t); + LIST_ADD(&proc_getReadyQueues(process)[priority], t); hal_spinlockClear(&threads_common.spinlock, &sc); @@ -636,7 +762,7 @@ int proc_threadCreate(process_t *process, startFn_t start, int *id, u8 priority, static u8 _proc_lockGetPriority(lock_t *lock) { - u8 priority = MAX_PRIO; + u8 priority = NUM_PRIO - 1U; thread_t *thread = lock->queue; if (thread != NULL) { @@ -654,7 +780,7 @@ static u8 _proc_lockGetPriority(lock_t *lock) static u8 _proc_threadGetLockPriority(thread_t *thread) { - u8 ret, priority = MAX_PRIO; + u8 ret, priority = NUM_PRIO - 1U; lock_t *lock = thread->locks; if (lock != NULL) { @@ -681,6 +807,8 @@ static u8 _proc_threadGetPriority(thread_t *thread) static void _proc_threadSetPriority(thread_t *thread, u8 priority) { unsigned int i; + thread_t **readyQueues; + /* Don't allow decreasing the priority below base level */ if (priority > thread->priorityBase) { @@ -695,11 +823,12 @@ static void _proc_threadSetPriority(thread_t *thread, u8 priority) } if (i == hal_cpuGetCount()) { - LIB_ASSERT(LIST_BELONGS(&threads_common.ready[thread->priority], thread) != 0, + readyQueues = proc_getReadyQueues(thread->process); + LIB_ASSERT(LIST_BELONGS(&readyQueues[thread->priority], thread) != 0, "thread: 0x%p, tid: %d, priority: %d, is not on the ready list", thread, proc_getTid(thread), thread->priority); - LIST_REMOVE(&threads_common.ready[thread->priority], thread); - LIST_ADD(&threads_common.ready[priority], thread); + LIST_REMOVE(&readyQueues[thread->priority], thread); + LIST_ADD(&readyQueues[priority], thread); } } @@ -719,7 +848,7 @@ int proc_threadPriority(int signedPriority) return -EINVAL; } - if ((signedPriority >= 0) && ((size_t)signedPriority >= sizeof(threads_common.ready) / sizeof(threads_common.ready[0]))) { + if ((signedPriority >= 0) && ((size_t)signedPriority >= NUM_PRIO)) { return -EINVAL; } @@ -879,7 +1008,7 @@ static void _proc_threadDequeue(thread_t *t) } if (t->wakeup != 0) { - lib_rbRemove(&threads_common.sleeping, &t->sleeplinkage); + lib_rbRemove(&threads_common.windows[proc_getSchedWindowId(t->process)]->sleeping, &t->sleeplinkage); } t->wakeup = 0; @@ -895,7 +1024,7 @@ static void _proc_threadDequeue(thread_t *t) } if (i == hal_cpuGetCount()) { - LIST_ADD(&threads_common.ready[t->priority], t); + LIST_ADD(&proc_getReadyQueues(t->process)[t->priority], t); } } @@ -919,9 +1048,7 @@ static void _proc_threadEnqueue(thread_t **queue, time_t timeout, u8 interruptib current->interruptible = interruptible & 0x1U; if (timeout != 0) { - current->wakeup = timeout; - (void)lib_rbInsert(&threads_common.sleeping, ¤t->sleeplinkage); - _threads_updateWakeup(_proc_gettimeRaw(), NULL); + _threads_sleepingInsert(current, timeout); } _threads_enqueued(current); @@ -947,19 +1074,19 @@ static int _proc_threadWait(thread_t **queue, time_t timeout, spinlock_ctx_t *sc static int _proc_threadSleepAbs(time_t abs, time_t now, spinlock_ctx_t *sc) { + thread_t *current; + /* Handle usleep(0) (yield) */ if (abs > now) { - thread_t *current = _proc_current(); + current = _proc_current(); current->state = SLEEP; current->wait = NULL; - current->wakeup = abs; current->interruptible = 1; - (void)lib_rbInsert(&threads_common.sleeping, ¤t->sleeplinkage); + _threads_sleepingInsert(current, abs); _threads_enqueued(current); - _threads_updateWakeup(now, NULL); } return hal_cpuReschedule(&threads_common.spinlock, sc); @@ -1255,21 +1382,69 @@ int proc_settime(time_t offs) static time_t _proc_nextWakeup(void) { - thread_t *thread; - time_t wakeup = 0; - time_t now; + thread_t **windowReady; + unsigned int i; + time_t now, timeToWindow, delayed, cycleStart; + time_t wakeup = NO_WAKEUP; + unsigned int actWindowIdx, window; + size_t windowIdx; + syspage_sched_cycle_t *cycle = threads_common.cycles[hal_cpuGetID()]; - thread = lib_treeof(thread_t, sleeplinkage, lib_rbMinimum(threads_common.sleeping.root)); - if (thread != NULL) { - now = _proc_gettimeRaw(); - if (now >= thread->wakeup) { - wakeup = 0; - } - else { - wakeup = thread->wakeup - now; + now = _proc_gettimeRaw(); + + if (threads_common.windows[cycle->bgId]->sleepMin != NO_WAKEUP) { + wakeup = threads_common.windows[cycle->bgId]->sleepMin - now; + if (wakeup <= 0) { + wakeup = 1; } } + if (threads_nonBackroundSchedWindows() == 0) { + return wakeup; + } + + cycleStart = now - (now % threads_schedWindowsCycleDuration()); + timeToWindow = cycleStart; + for (actWindowIdx = 0; now % threads_schedWindowsCycleDuration() >= cycle->windows[actWindowIdx].stop; actWindowIdx++) { + timeToWindow = cycleStart + cycle->windows[actWindowIdx].stop; + } + + windowIdx = actWindowIdx; + do { + window = cycle->windows[windowIdx].id; + + if ((wakeup != NO_WAKEUP) && (timeToWindow >= wakeup)) { + break; + } + + /* check if there's anything to schedule */ + windowReady = threads_common.windows[window]->ready; + for (i = 0; i < NUM_PRIO; ++i) { + if (windowReady[i] != NULL) { + wakeup = timeToWindow; + break; + } + } + + /* check first wakeup in this window */ + if (threads_common.windows[window]->sleepMin != NO_WAKEUP) { + delayed = threads_common.windows[window]->sleepMin - now; + if (delayed < timeToWindow) { + delayed = timeToWindow; + } + if (delayed < 0) { + delayed = 1; + } + + if ((wakeup == NO_WAKEUP) || (delayed < wakeup)) { + wakeup = delayed; + } + } + + timeToWindow = cycleStart + cycle->windows[windowIdx].stop; + windowIdx = (windowIdx + 1U) % cycle->len; + } while (windowIdx != actWindowIdx); + return wakeup; } @@ -1869,6 +2044,7 @@ void proc_threadsDump(u8 priority) { thread_t *t; spinlock_ctx_t sc; + size_t window; /* Strictly needed - no lock can be taken * while threads_common.spinlock is being @@ -1878,16 +2054,18 @@ void proc_threadsDump(u8 priority) lib_printf("threads: "); hal_spinlockSet(&threads_common.spinlock, &sc); - t = threads_common.ready[priority]; - do { - lib_printf("[%p] ", t); + for (window = 0U; window < syspage_schedulerConfig()->windowCnt; ++window) { + t = threads_common.windows[window]->ready[priority]; + do { + lib_printf("[%p(%u)] ", t, window); - if (t == NULL) { - break; - } + if (t == NULL) { + break; + } - t = t->next; - } while (t != threads_common.ready[priority]); + t = t->next; + } while (t != threads_common.windows[window]->ready[priority]); + } hal_spinlockClear(&threads_common.spinlock, &sc); lib_printf("\n"); @@ -2023,7 +2201,8 @@ int proc_threadsList(int n, threadinfo_t *info) int _threads_init(vm_map_t *kmap, vm_object_t *kernel) { - unsigned int i; + unsigned int i, j; + size_t cnt; threads_common.kmap = kmap; threads_common.ghosts = NULL; threads_common.reaper = NULL; @@ -2037,15 +2216,33 @@ int _threads_init(vm_map_t *kmap, vm_object_t *kernel) threads_common.stackCanary[i] = ((i & 1U) != 0U) ? 0xaaU : 0x55U; } - /* Initiaizlie scheduler queue */ - for (i = 0; i < sizeof(threads_common.ready) / sizeof(thread_t *); i++) { - threads_common.ready[i] = NULL; + /* Initialize scheduler queues and sleeping trees for each scheduling window */ + LIB_ASSERT_ALWAYS(((syspage_schedulerConfig()->flags & (unsigned int)sFlagCommonCycle) != 0U) || (syspage_schedulerConfig()->cycleCnt == hal_cpuGetCount()), + "syspage scheduler configuration mismatches actual CPU count (%u vs %u)", + syspage_schedulerConfig()->cycleCnt, hal_cpuGetCount()); + + cnt = syspage_schedulerConfig()->windowCnt + 1U; /* +1 for kernel threads window */ + threads_common.windows = vm_kmalloc(sizeof(sched_window_t *) * cnt); + if (threads_common.windows == NULL) { + return -ENOMEM; + } + /* Merge windows that share a partition */ + for (i = 0; i < cnt; i++) { + threads_common.windows[i] = vm_kmalloc(sizeof(sched_window_t)); + if (threads_common.windows[i] == NULL) { + return -ENOMEM; + } + for (j = 0; j < NUM_PRIO; j++) { + threads_common.windows[i]->ready[j] = NULL; + } + lib_rbInit(&threads_common.windows[i]->sleeping, threads_sleepcmp, NULL); + threads_common.windows[i]->sleepMin = NO_WAKEUP; + threads_common.windows[i]->sleepMinClaimed = (unsigned int)-1; } - lib_rbInit(&threads_common.sleeping, threads_sleepcmp, NULL); lib_idtreeInit(&threads_common.id); - lib_printf("proc: Initializing thread scheduler, priorities=%d\n", sizeof(threads_common.ready) / sizeof(thread_t *)); + lib_printf("proc: Initializing thread scheduler, priorities=%d\n", NUM_PRIO); hal_spinlockCreate(&threads_common.spinlock, "threads.spinlock"); @@ -2059,7 +2256,26 @@ int _threads_init(vm_map_t *kmap, vm_object_t *kernel) /* Run idle thread on every cpu */ for (i = 0; i < hal_cpuGetCount(); i++) { threads_common.current[i] = NULL; - (void)proc_threadCreate(NULL, threads_idlethr, NULL, MAX_PRIO, (size_t)SIZE_KSTACK, NULL, 0, 0, NULL); + (void)proc_threadCreate(NULL, threads_idlethr, NULL, NUM_PRIO - 1U, (size_t)SIZE_KSTACK, NULL, 0, 0, NULL); + } + + /* Initialize structures for scheduler windows switching */ + threads_common.actWindow = vm_kmalloc(sizeof(*threads_common.actWindow) * hal_cpuGetCount()); + if (threads_common.actWindow == NULL) { + return -ENOMEM; + } + threads_common.cycles = vm_kmalloc(sizeof(*threads_common.cycles) * hal_cpuGetCount()); + if (threads_common.cycles == NULL) { + return -ENOMEM; + } + for (i = 0; i < hal_cpuGetCount(); i++) { + threads_common.actWindow[i] = 0U; + if ((syspage_schedulerConfig()->flags & (unsigned int)sFlagCommonCycle) != 0U) { + threads_common.cycles[i] = syspage_schedulerConfig()->cycles[0]; + } + else { + threads_common.cycles[i] = syspage_schedulerConfig()->cycles[i]; + } } /* Install scheduler on clock interrupt */ @@ -2073,5 +2289,7 @@ int _threads_init(vm_map_t *kmap, vm_object_t *kernel) hal_memset(&threads_common.timeintrHandler, 0, sizeof(threads_common.timeintrHandler)); (void)hal_timerRegister(threads_timeintr, NULL, &threads_common.timeintrHandler); + /* workaround for sparcv8leon which does not set up SYSTICK timer */ + hal_timerSetWakeup((unsigned int)SYSTICK_INTERVAL); return EOK; } diff --git a/syscalls.c b/syscalls.c index 7387da10e..428973170 100644 --- a/syscalls.c +++ b/syscalls.c @@ -102,7 +102,7 @@ int syscalls_sys_mmap(u8 *ustack) if (err < 0) { return err; } - err = vm_objectGet(&o, oid); + err = vm_objectGet(&o, oid, proc->partition); if (err < 0) { return err; } diff --git a/syspage.c b/syspage.c index 795c36355..b797e3787 100644 --- a/syspage.c +++ b/syspage.c @@ -183,11 +183,25 @@ void syspage_progShow(void) } +syspage_sched_t *syspage_schedulerConfig(void) +{ + return syspage_common.syspage->sched; +} + + +syspage_part_t *syspage_partitionList(void) +{ + return syspage_common.syspage->partitions; +} + + void syspage_init(void) { syspage_prog_t *prog; + syspage_part_t *part; syspage_map_t *map; mapent_t *entry; + unsigned int i; syspage_common.syspage = (syspage_t *)hal_syspageAddr(); @@ -225,7 +239,31 @@ void syspage_init(void) prog->dmaps = hal_syspageRelocate(prog->dmaps); prog->imaps = hal_syspageRelocate(prog->imaps); prog->argv = hal_syspageRelocate(prog->argv); + prog->partition = hal_syspageRelocate(prog->partition); + prog->hal = hal_syspageRelocate(prog->hal); prog = prog->next; } while (prog != syspage_common.syspage->progs); } + + /* Partition's relocation */ + LIB_ASSERT_ALWAYS(syspage_common.syspage->partitions != NULL, "No partitions in syspage"); + syspage_common.syspage->partitions = hal_syspageRelocate(syspage_common.syspage->partitions); + part = syspage_common.syspage->partitions; + + do { + part->next = hal_syspageRelocate(part->next); + part->prev = hal_syspageRelocate(part->prev); + + part->name = hal_syspageRelocate(part->name); + part->hal = hal_syspageRelocate(part->hal); + + part = part->next; + } while (part != syspage_common.syspage->partitions); + + /* Scheduler configuration relocation */ + LIB_ASSERT_ALWAYS(syspage_common.syspage->sched != NULL, "Missing scheduler configuration in syspage"); + syspage_common.syspage->sched = hal_syspageRelocate(syspage_common.syspage->sched); + for (i = 0; i < syspage_common.syspage->sched->cycleCnt; ++i) { + syspage_common.syspage->sched->cycles[i] = hal_syspageRelocate(syspage_common.syspage->sched->cycles[i]); + } } diff --git a/syspage.h b/syspage.h index f587ae7d4..0a82e15aa 100644 --- a/syspage.h +++ b/syspage.h @@ -51,6 +51,16 @@ const syspage_prog_t *syspage_progIdResolve(unsigned int id); const syspage_prog_t *syspage_progNameResolve(const char *name); +/* Scheduler configuration */ + +syspage_sched_t *syspage_schedulerConfig(void); + + +/* Partition configuration */ + +syspage_part_t *syspage_partitionList(void); + + /* General functions */ void syspage_progShow(void); diff --git a/test/proc.c b/test/proc.c index 05ec232ea..0b473c40b 100644 --- a/test/proc.c +++ b/test/proc.c @@ -202,7 +202,7 @@ static void test_proc_initthr(void *arg) void test_proc_exit(void) { - proc_start(test_proc_initthr, NULL, (const char *)"init"); + proc_start(test_proc_initthr, NULL, (const char *)"init", NULL); hal_cpuEnableInterrupts(); hal_cpuReschedule(NULL, NULL); diff --git a/test/vm.c b/test/vm.c index 3a3b80aea..cf7d2b486 100644 --- a/test/vm.c +++ b/test/vm.c @@ -44,7 +44,7 @@ void test_vm_alloc(void) maxsize = max(maxsize, size); hal_cpuGetCycles(&b); - p = vm_pageAlloc(size, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP); + p = vm_pageAlloc(NULL, size, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP, NULL); hal_cpuGetCycles(&e); if (p == NULL) { @@ -52,7 +52,7 @@ void test_vm_alloc(void) break; } - vm_pageFree(p); + vm_pageFree(p, NULL); lib_printf("\rtest: size=%d, n=%d", size, n); @@ -115,7 +115,7 @@ void test_vm_kmalloc(void) vm_kmallocGetStats(&kmallocsz); vm_mapGetStats(&mapallocsz); - vm_pageGetStats(&freesz); + _vm_pageGetStats(&freesz); lib_printf("test: Testing kmalloc, kmalloc=%d, map=%d, free=%dKB\n", kmallocsz, mapallocsz, freesz / 1024U); @@ -150,7 +150,7 @@ void test_vm_kmalloc(void) vm_kmallocGetStats(&kmallocsz); vm_mapGetStats(&mapallocsz); - vm_pageGetStats(&freesz); + _vm_pageGetStats(&freesz); lib_printf("test: Memory after test, kmalloc=%d, map=%d, free=%dKB\n", kmallocsz, mapallocsz, freesz / 1024U); // vm_mapDumpArenas(); diff --git a/vm/amap.c b/vm/amap.c index 7f68ae07d..0169277e9 100644 --- a/vm/amap.c +++ b/vm/amap.c @@ -28,7 +28,7 @@ static struct { } amap_common; -static anon_t *amap_putanon(anon_t *a) +static anon_t *amap_putanon(anon_t *a, struct _partition_t *part) { if (a == NULL) { return NULL; @@ -39,7 +39,7 @@ static anon_t *amap_putanon(anon_t *a) return a; } - vm_pageFree(a->page); + vm_pageFree(a->page, part); (void)proc_lockClear(&a->lock); (void)proc_lockDone(&a->lock); vm_kfree(a); @@ -57,7 +57,7 @@ void amap_putanons(amap_t *amap, size_t offset, size_t size) (void)proc_lockSet(&amap->lock); for (i = offset / SIZE_PAGE; i < (offset + size) / SIZE_PAGE; ++i) { - (void)amap_putanon(amap->anons[i]); + (void)amap_putanon(amap->anons[i], amap->partition); } (void)proc_lockClear(&amap->lock); } @@ -107,7 +107,7 @@ amap_t *amap_ref(amap_t *amap) } -amap_t *amap_create(amap_t *amap, size_t *offset, size_t size) +amap_t *amap_create(amap_t *amap, size_t *offset, size_t size, partition_t *part) { size_t i = size / SIZE_PAGE; amap_t *new; @@ -137,6 +137,7 @@ amap_t *amap_create(amap_t *amap, size_t *offset, size_t size) (void)proc_lockInit(&new->lock, &proc_lockAttrDefault, "amap.map"); new->size = i; new->refs = 1; + new->partition = part; *offset = *offset / SIZE_PAGE; @@ -272,7 +273,7 @@ int amap_page(vm_map_t *map, amap_t *amap, vm_object_t *o, void *vaddr, size_t a if (a != NULL || o != NULL) { /* Copy from object or shared anon */ - *page = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); + *page = vm_pageAlloc(map->phMaps, SIZE_PAGE, PAGE_OWNER_APP, amap->partition); if (*page == NULL) { (void)amap_unmap(map, v); if (a != NULL) { @@ -283,7 +284,7 @@ int amap_page(vm_map_t *map, amap_t *amap, vm_object_t *o, void *vaddr, size_t a } w = amap_map(map, *page); if (w == NULL) { - vm_pageFree(*page); + vm_pageFree(*page, amap->partition); *page = NULL; (void)amap_unmap(map, v); if (a != NULL) { @@ -308,7 +309,7 @@ int amap_page(vm_map_t *map, amap_t *amap, vm_object_t *o, void *vaddr, size_t a amap->anons[aoffs / SIZE_PAGE] = anon_new(*page); if (amap->anons[aoffs / SIZE_PAGE] == NULL) { - vm_pageFree(*page); + vm_pageFree(*page, amap->partition); *page = NULL; err = -ENOMEM; } diff --git a/vm/amap.h b/vm/amap.h index 5f9dd1892..7693ea3c5 100644 --- a/vm/amap.h +++ b/vm/amap.h @@ -21,6 +21,7 @@ struct _vm_map_t; struct _vm_object_t; +struct _partition_t; typedef struct _anon_t { @@ -32,6 +33,7 @@ typedef struct _anon_t { typedef struct _amap_t { lock_t lock; + struct _partition_t *partition; size_t size; int refs; anon_t *anons[]; @@ -50,7 +52,7 @@ void amap_putanons(amap_t *amap, size_t offset, size_t size); void amap_getanons(amap_t *amap, size_t offset, size_t size); -amap_t *amap_create(amap_t *amap, size_t *offset, size_t size); +amap_t *amap_create(amap_t *amap, size_t *offset, size_t size, struct _partition_t *part); void amap_put(amap_t *amap); diff --git a/vm/map.c b/vm/map.c index b914923ff..18d986439 100644 --- a/vm/map.c +++ b/vm/map.c @@ -51,7 +51,7 @@ static map_entry_t *map_allocN(size_t n); void map_free(map_entry_t *entry); -static int _map_force(vm_map_t *map, map_entry_t *e, void *paddr, vm_prot_t prot); +static int _map_force(vm_map_t *map, map_entry_t *e, void *paddr, vm_prot_t prot, partition_t *part); static int map_cmp(rbnode_t *n1, rbnode_t *n2) @@ -257,141 +257,161 @@ static void *_map_map(vm_map_t *map, void *vaddr, process_t *proc, size_t size, /* Return requested offset as a physical address only if it fits into ptr_t (otherwise it makes no sense) */ return offs <= (ptr_t)-1 ? (void *)(ptr_t)offs : NULL; } -#endif - - v = _map_find(map, vaddr, size, &prev, &next); - if (v == NULL) { - return NULL; + if ((proc != NULL) && (proc->partition != NULL)) { + (void)proc_lockSet(&proc->partition->lock); + if ((proc->partition->config->availableMem < proc->partition->usedMem) || + (size > proc->partition->config->availableMem - proc->partition->usedMem)) { + (void)proc_lockClear(&proc->partition->lock); + return NULL; + } } +#endif + do { + v = _map_find(map, vaddr, size, &prev, &next); + if (v == NULL) { + break; + } - rmerge = (next != NULL && v + size == next->vaddr && next->object == o && next->flags == flags && next->prot == prot && next->protOrig == prot) ? 1U : 0U; - lmerge = (prev != NULL && v == prev->vaddr + prev->size && prev->object == o && prev->flags == flags && prev->prot == prot && prev->protOrig == prot) ? 1U : 0U; + rmerge = (next != NULL && v + size == next->vaddr && next->object == o && next->flags == flags && next->prot == prot && next->protOrig == prot) ? 1U : 0U; + lmerge = (prev != NULL && v == prev->vaddr + prev->size && prev->object == o && prev->flags == flags && prev->prot == prot && prev->protOrig == prot) ? 1U : 0U; - if (offs != VM_OFFS_MAX) { - if ((offs & (SIZE_PAGE - 1UL)) != 0UL) { - return NULL; - } + if (offs != VM_OFFS_MAX) { + if ((offs & (SIZE_PAGE - 1UL)) != 0UL) { + v = NULL; + break; + } - if ((rmerge != 0U) && (next->offs != offs + size)) { - rmerge = 0U; - } + if ((rmerge != 0U) && (next->offs != offs + size)) { + rmerge = 0U; + } - if ((lmerge != 0U) && (offs != prev->offs + prev->size)) { - lmerge = 0U; + if ((lmerge != 0U) && (offs != prev->offs + prev->size)) { + lmerge = 0U; + } } - } #ifdef NOMMU - rmerge = (rmerge != 0U && (proc == next->process)) ? 1U : 0U; - lmerge = (lmerge != 0U && (proc == prev->process)) ? 1U : 0U; + rmerge = (rmerge != 0U && (proc == next->process)) ? 1U : 0U; + lmerge = (lmerge != 0U && (proc == prev->process)) ? 1U : 0U; #endif - if (o == NULL) { - if (lmerge != 0U && rmerge != 0U && (next->amap == prev->amap)) { - /* Both use the same amap, can merge */ - } - else { - /* Can't merge to the left if amap array size is too small */ - if (lmerge != 0U) { - amap = prev->amap; - if (amap != NULL && (amap->size * SIZE_PAGE - prev->aoffs - prev->size) < size) { - lmerge = 0; - } + if (o == NULL) { + if (lmerge != 0U && rmerge != 0U && (next->amap == prev->amap)) { + /* Both use the same amap, can merge */ } - /* Can't merge to the right if amap offset is too small */ - if (rmerge != 0U) { - amap = next->amap; - if (amap != NULL && next->aoffs < size) { + else { + /* Can't merge to the left if amap array size is too small */ + if (lmerge != 0U) { + amap = prev->amap; + if (amap != NULL && (amap->size * SIZE_PAGE - prev->aoffs - prev->size) < size) { + lmerge = 0; + } + } + /* Can't merge to the right if amap offset is too small */ + if (rmerge != 0U) { + amap = next->amap; + if (amap != NULL && next->aoffs < size) { + rmerge = 0; + } + } + /* amaps differ, we can only merge one way */ + if (lmerge != 0U && rmerge != 0U) { rmerge = 0; } } - /* amaps differ, we can only merge one way */ - if (lmerge != 0U && rmerge != 0U) { - rmerge = 0; - } } - } - if (rmerge != 0U && lmerge != 0U) { - e = prev; - e->size += size + next->size; - e->rmaxgap = next->rmaxgap; + if (rmerge != 0U && lmerge != 0U) { + e = prev; + e->size += size + next->size; + e->rmaxgap = next->rmaxgap; - map_augment(&e->linkage); - _entry_put(map, next); - } - else if (rmerge != 0U) { - e = next; - e->vaddr = v; - e->offs = offs; - e->size += size; - e->lmaxgap -= size; - - if (e->aoffs != 0U) { - e->aoffs -= size; - } - - if (prev != NULL) { - prev->rmaxgap -= size; - map_augment(&prev->linkage); + map_augment(&e->linkage); + _entry_put(map, next); } + else if (rmerge != 0U) { + e = next; + e->vaddr = v; + e->offs = offs; + e->size += size; + e->lmaxgap -= size; + + if (e->aoffs != 0U) { + e->aoffs -= size; + } - map_augment(&e->linkage); - } - else if (lmerge != 0U) { - e = prev; - e->size += size; - e->rmaxgap -= size; + if (prev != NULL) { + prev->rmaxgap -= size; + map_augment(&prev->linkage); + } - if (next != NULL) { - next->lmaxgap -= size; - map_augment(&next->linkage); + map_augment(&e->linkage); } + else if (lmerge != 0U) { + e = prev; + e->size += size; + e->rmaxgap -= size; + + if (next != NULL) { + next->lmaxgap -= size; + map_augment(&next->linkage); + } - map_augment(&e->linkage); - } - else { - e = map_alloc(); - if (e == NULL) { - return NULL; + map_augment(&e->linkage); } + else { + e = map_alloc(); + if (e == NULL) { + v = NULL; + break; + } - e->vaddr = v; - e->size = size; - e->object = vm_objectRef(o); - e->offs = offs; - e->flags = flags; - e->prot = prot; - e->protOrig = prot; + e->vaddr = v; + e->size = size; + e->object = vm_objectRef(o); + e->offs = offs; + e->flags = flags; + e->prot = prot; + e->protOrig = prot; - e->amap = NULL; - e->aoffs = 0; + e->amap = NULL; + e->aoffs = 0; - if (o == NULL) { - /* Try to use existing amap */ - if (next != NULL && next->amap != NULL && e->vaddr >= (next->vaddr - next->aoffs)) { - e->amap = amap_ref(next->amap); - e->aoffs = next->aoffs - ((ptr_t)next->vaddr - (ptr_t)e->vaddr); - } - else if (prev != NULL && prev->amap != NULL && (SIZE_PAGE * prev->amap->size - prev->aoffs + (size_t)prev->vaddr) >= ((size_t)e->vaddr + size)) { - e->amap = amap_ref(prev->amap); - e->aoffs = prev->aoffs + ((ptr_t)e->vaddr - (ptr_t)prev->vaddr); - } - else { - /* No action required */ + if (o == NULL) { + /* Try to use existing amap */ + if (next != NULL && next->amap != NULL && e->vaddr >= (next->vaddr - next->aoffs)) { + e->amap = amap_ref(next->amap); + e->aoffs = next->aoffs - ((ptr_t)next->vaddr - (ptr_t)e->vaddr); + } + else if (prev != NULL && prev->amap != NULL && (SIZE_PAGE * prev->amap->size - prev->aoffs + (size_t)prev->vaddr) >= ((size_t)e->vaddr + size)) { + e->amap = amap_ref(prev->amap); + e->aoffs = prev->aoffs + ((ptr_t)e->vaddr - (ptr_t)prev->vaddr); + } + else { + /* No action required */ + } } + + (void)_map_add(proc, map, e); } - (void)_map_add(proc, map, e); - } + /* Clear anon entries */ + if (e->amap != NULL) { + amap_clear(e->amap, e->aoffs + ((ptr_t)v - (ptr_t)e->vaddr), size); + } + if (entry != NULL) { + *entry = e; + } + } while (0); - /* Clear anon entries */ - if (e->amap != NULL) { - amap_clear(e->amap, e->aoffs + ((ptr_t)v - (ptr_t)e->vaddr), size); - } - if (entry != NULL) { - *entry = e; +#ifdef NOMMU + if ((proc != NULL) && (proc->partition != NULL)) { + if (v != NULL) { + proc->partition->usedMem += size; + } + (void)proc_lockClear(&proc->partition->lock); } +#endif return v; } @@ -399,10 +419,27 @@ static void *_map_map(vm_map_t *map, void *vaddr, process_t *proc, size_t size, void *vm_mapFind(vm_map_t *map, void *vaddr, size_t size, vm_flags_t flags, vm_prot_t prot) { +#ifdef NOMMU + unsigned int i; + if (map->phMaps == NULL) { + (void)proc_lockSet(&map->lock); + vaddr = _map_map(map, vaddr, NULL, size, prot, map_common.kernel, VM_OFFS_MAX, flags, NULL); + (void)proc_lockClear(&map->lock); + return vaddr; + } + for (i = 0; map->phMaps[i] != NULL; i++) { + (void)proc_lockSet(&map->phMaps[i]->lock); + vaddr = _map_map(map->phMaps[i], vaddr, NULL, size, prot, map_common.kernel, VM_OFFS_MAX, flags, NULL); + (void)proc_lockClear(&map->phMaps[i]->lock); + if (vaddr != NULL) { + return vaddr; + } + } +#else (void)proc_lockSet(&map->lock); vaddr = _map_map(map, vaddr, NULL, size, prot, map_common.kernel, VM_OFFS_MAX, flags, NULL); (void)proc_lockClear(&map->lock); - +#endif return vaddr; } @@ -529,12 +566,44 @@ int _vm_munmap(vm_map_t *map, void *vaddr, size_t size) if (putEntry != 0) { _entry_put(map, e); } +#ifdef NOMMU + if ((proc != NULL) && (proc->partition != NULL)) { + (void)proc_lockSet(&proc->partition->lock); + proc->partition->usedMem -= overlapSize; + (void)proc_lockClear(&proc->partition->lock); + } +#endif } return EOK; } +static int _vm_munmapPhMaps(vm_map_t *map, void *vaddr, size_t size) +{ +#ifdef NOMMU + unsigned int i; + int err; + if (map->phMaps == NULL) { + return _vm_munmap(map, vaddr, size); + } + for (i = 0; map->phMaps[i] != NULL; i++) { + if (((ptr_t)map->phMaps[i]->start < (ptr_t)vaddr + size) && ((ptr_t)map->phMaps[i]->stop >= (ptr_t)vaddr)) { + (void)proc_lockSet(&map->phMaps[i]->lock); + err = _vm_munmap(map->phMaps[i], vaddr, size); + (void)proc_lockClear(&map->phMaps[i]->lock); + if (err != EOK) { + return err; + } + } + } + return EOK; +#else + return _vm_munmap(map, vaddr, size); +#endif +} + + vm_attr_t vm_flagsToAttr(vm_flags_t flags) { vm_attr_t attr = 0; @@ -623,7 +692,7 @@ void *_vm_mmap(vm_map_t *map, void *vaddr, page_t *p, size_t size, vm_prot_t pro } for (w = vaddr; w < vaddr + size; w += SIZE_PAGE) { - if (_map_force(map, e, w, prot) != 0) { + if (_map_force(map, e, w, prot, process != NULL ? process->partition : NULL) != 0) { (void)_vm_munmap(map, vaddr, size); return NULL; } @@ -639,9 +708,27 @@ void *vm_mmap(vm_map_t *map, void *vaddr, page_t *p, size_t size, vm_prot_t prot map = map_common.kmap; } +#ifdef NOMMU + unsigned int i; + if (map->phMaps == NULL) { + (void)proc_lockSet(&map->lock); + vaddr = _vm_mmap(map, vaddr, p, size, prot, o, (offs < 0) ? VM_OFFS_MAX : (u64)offs, flags); + (void)proc_lockClear(&map->lock); + return vaddr; + } + for (i = 0; map->phMaps[i] != NULL; i++) { + (void)proc_lockSet(&map->phMaps[i]->lock); + vaddr = _vm_mmap(map->phMaps[i], vaddr, p, size, prot, o, (offs < 0) ? VM_OFFS_MAX : (u64)offs, flags); + (void)proc_lockClear(&map->phMaps[i]->lock); + if (vaddr != NULL) { + return vaddr; + } + } +#else (void)proc_lockSet(&map->lock); vaddr = _vm_mmap(map, vaddr, p, size, prot, o, (offs < 0) ? VM_OFFS_MAX : (u64)offs, flags); (void)proc_lockClear(&map->lock); +#endif return vaddr; } @@ -705,6 +792,10 @@ int vm_mapForce(vm_map_t *map, void *paddr, vm_prot_t prot) { map_entry_t t, *e; int err; + partition_t *part = NULL; + if (proc_current()->process != NULL) { + part = proc_current()->process->partition; + } (void)proc_lockSet(&map->lock); @@ -718,7 +809,7 @@ int vm_mapForce(vm_map_t *map, void *paddr, vm_prot_t prot) return -EFAULT; } - err = _map_force(map, e, paddr, prot); + err = _map_force(map, e, paddr, prot, part); (void)proc_lockClear(&map->lock); return err; } @@ -730,7 +821,7 @@ static vm_prot_t map_checkProt(vm_prot_t baseProt, vm_prot_t newProt) } -static int _map_force(vm_map_t *map, map_entry_t *e, void *paddr, vm_prot_t prot) +static int _map_force(vm_map_t *map, map_entry_t *e, void *paddr, vm_prot_t prot, partition_t *part) { vm_attr_t attr; size_t offs; @@ -744,7 +835,7 @@ static int _map_force(vm_map_t *map, map_entry_t *e, void *paddr, vm_prot_t prot return -EINVAL; } if ((((prot & PROT_WRITE) != 0U) && ((e->flags & MAP_NEEDSCOPY) != 0U)) || ((e->object == NULL) && (e->amap == NULL))) { - amapNew = amap_create(e->amap, &e->aoffs, e->size); + amapNew = amap_create(e->amap, &e->aoffs, e->size, part); if (amapNew == NULL) { return -ENOMEM; } @@ -843,7 +934,7 @@ int vm_munmap(vm_map_t *map, void *vaddr, size_t size) int result; (void)proc_lockSet(&map->lock); - result = _vm_munmap(map, vaddr, size); + result = _vm_munmapPhMaps(map, vaddr, size); (void)proc_lockClear(&map->lock); return result; @@ -965,7 +1056,7 @@ int vm_mprotect(vm_map_t *map, void *vaddr, size_t len, vm_prot_t prot) } } else { - result = _map_force(map, e, currVaddr, prot); + result = _map_force(map, e, currVaddr, prot, p != NULL ? p->partition : NULL); } } @@ -992,28 +1083,30 @@ void vm_mapDump(vm_map_t *map) } -int vm_mapCreate(vm_map_t *map, void *start, void *stop) +int vm_mapCreate(vm_map_t *map, void *start, void *stop, ph_map_t **phMaps) { map->start = start; map->stop = stop; map->pmap.start = start; map->pmap.end = stop; + map->phMaps = phMaps; + #ifndef NOMMU - map->pmap.pmapp = vm_pageAlloc(SIZE_PDIR, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE); + map->pmap.pmapp = vm_pageAlloc(map_common.kmap->phMaps, SIZE_PDIR, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE, NULL); if (map->pmap.pmapp == NULL) { return -ENOMEM; } map->pmap.pmapv = vm_mmap(map_common.kmap, NULL, map->pmap.pmapp, 1UL << map->pmap.pmapp->idx, PROT_READ | PROT_WRITE, map_common.kernel, -1, MAP_NONE); if (map->pmap.pmapv == NULL) { - vm_pageFree(map->pmap.pmapp); + vm_pageFree(map->pmap.pmapp, NULL); return -ENOMEM; } - (void)pmap_create(&map->pmap, &map_common.kmap->pmap, map->pmap.pmapp, map->pmap.pmapv); + (void)pmap_create(&map->pmap, &map_common.kmap->pmap, map->pmap.pmapp, NULL, map->pmap.pmapv); #else - (void)pmap_create(&map->pmap, &map_common.kmap->pmap, NULL, NULL); + (void)pmap_create(&map->pmap, &map_common.kmap->pmap, NULL, NULL, NULL); #endif (void)proc_lockInit(&map->lock, &proc_lockAttrDefault, "map.map"); @@ -1038,35 +1131,34 @@ void map_free(map_entry_t *entry) } -void vm_mapDestroy(process_t *p, vm_map_t *map) +ph_map_t **vm_createPhMapList(const unsigned char *maps, size_t mapSz) { - map_entry_t *e; + ph_map_t **phMaps; + size_t i; -#ifndef NOMMU - addr_t a; - rbnode_t *n; - unsigned int i = 0; + phMaps = vm_kmalloc(sizeof(ph_map_t *) * (mapSz + 1U)); + if (phMaps == NULL) { + return NULL; + } - for (;;) { - a = pmap_destroy(&map->pmap, &i); - if (a == 0U) { - break; + for (i = 0; i < mapSz; i++) { + phMaps[i] = vm_getPhysicalMap((int)maps[i]); + if (phMaps[i] == NULL) { + vm_kfree(phMaps); + return NULL; } - vm_pageFree(page_get(a)); } + phMaps[mapSz] = NULL; - (void)vm_munmap(map_common.kmap, map->pmap.pmapv, SIZE_PDIR); - vm_pageFree(map->pmap.pmapp); + return phMaps; +} - for (n = map->tree.root; n != NULL; n = map->tree.root) { - e = lib_treeof(map_entry_t, linkage, n); - amap_putanons(e->amap, e->aoffs, e->size); - _entry_put(map, e); - } - (void)proc_lockDone(&map->lock); -#else - map_entry_t *temp = NULL; +#ifdef NOMMU +static size_t vm_phMapDestroy(process_t *p, ph_map_t *map) +{ + map_entry_t *e, *temp = NULL; + size_t freed = 0U; (void)proc_lockSet2(&map->lock, &p->lock); @@ -1079,6 +1171,7 @@ void vm_mapDestroy(process_t *p, vm_map_t *map) LIST_ADD(&temp, e); } else { + freed += e->size; amap_put(e->amap); (void)vm_objectPut(e->object); lib_rbRemove(&map->tree, &e->linkage); @@ -1097,6 +1190,52 @@ void vm_mapDestroy(process_t *p, vm_map_t *map) (void)proc_lockClear(&p->lock); (void)proc_lockClear(&map->lock); + + return freed; +} +#endif + + +void vm_mapDestroy(process_t *p, vm_map_t *map) +{ + unsigned int i = 0; + +#ifndef NOMMU + addr_t a; + rbnode_t *n; + map_entry_t *e; + + for (;;) { + a = pmap_destroy(&map->pmap, &i); + if (a == 0U) { + break; + } + vm_pageFree(page_get(a), NULL); + } + + (void)vm_munmap(map_common.kmap, map->pmap.pmapv, SIZE_PDIR); + vm_pageFree(map->pmap.pmapp, NULL); + + for (n = map->tree.root; n != NULL; n = map->tree.root) { + e = lib_treeof(map_entry_t, linkage, n); + amap_putanons(e->amap, e->aoffs, e->size); + _entry_put(map, e); + } + + (void)proc_lockDone(&map->lock); +#else + size_t freed = 0U; + if (map->phMaps == NULL) { + freed += vm_phMapDestroy(p, map); + } + else { + for (i = 0; map->phMaps[i] != NULL; i++) { + freed += vm_phMapDestroy(p, map->phMaps[i]); + } + } + (void)proc_lockSet(&p->partition->lock); + p->partition->usedMem -= freed; + (void)proc_lockClear(&p->partition->lock); #endif } @@ -1158,7 +1297,7 @@ int vm_mapCopy(process_t *proc, vm_map_t *dst, vm_map_t *src) if ((proc == NULL) || (proc->lazy == 0U)) { for (offs = 0; offs < f->size; offs += SIZE_PAGE) { - err = _map_force(dst, f, (void *)((ptr_t)f->vaddr + offs), f->prot); + err = _map_force(dst, f, (void *)((ptr_t)f->vaddr + offs), f->prot, proc != NULL ? proc->partition : NULL); if (err != EOK) { (void)proc_lockClear(&dst->lock); (void)proc_lockClear(&src->lock); @@ -1230,11 +1369,13 @@ void vm_mapinfo(meminfo_t *info) rbnode_t *n; map_entry_t *e; vm_map_t *map; - const syspage_map_t *spMap; int size; process_t *process; size_t i; +#ifdef NOMMU + const syspage_map_t *spMap; size_t total, free; +#endif (void)proc_lockSet(&map_common.lock); @@ -1392,9 +1533,10 @@ void vm_mapinfo(meminfo_t *info) if (info->maps.mapsz != -1) { info->maps.total = 0; info->maps.free = 0; - - for (size = 0; (unsigned int)size < map_common.mapssz; ++size) { - map = vm_getSharedMap(size); + size = 0; +#ifdef NOMMU + for (; (unsigned int)size < map_common.mapssz; ++size) { + map = vm_getPhysicalMap(size); if (map == NULL) { if (size < info->maps.mapsz) { /* Store info that the map doesn't exist */ @@ -1444,7 +1586,7 @@ void vm_mapinfo(meminfo_t *info) } } } - +#endif info->maps.mapsz = size; } } @@ -1499,9 +1641,10 @@ void vm_mapGetStats(size_t *allocsz) } -vm_map_t *vm_getSharedMap(int map) +#ifdef NOMMU +ph_map_t *vm_getPhysicalMap(int map) { - vm_map_t *ret = NULL; + ph_map_t *ret = NULL; if (map >= 0 && (size_t)map < map_common.mapssz) { ret = map_common.maps[map]; @@ -1509,6 +1652,7 @@ vm_map_t *vm_getSharedMap(int map) return ret; } +#endif static int _map_mapsInit(vm_map_t *kmap, vm_object_t *kernel, void **bss, void **top) @@ -1553,7 +1697,7 @@ static int _map_mapsInit(vm_map_t *kmap, vm_object_t *kernel, void **bss, void * } map_common.maps[id] = (*bss); - if (vm_mapCreate(map_common.maps[id], (void *)map->start, (void *)map->end) < 0) { + if (vm_mapCreate(map_common.maps[id], (void *)map->start, (void *)map->end, NULL) < 0) { return -ENOMEM; } @@ -1623,7 +1767,7 @@ int _map_init(vm_map_t *kmap, vm_object_t *kernel, void **bss, void **top) map_common.kmap = kmap; map_common.kernel = kernel; - vm_pageGetStats(&freesz); + _vm_pageGetStats(&freesz); /* Init map entry pool */ map_common.ntotal = freesz / (3U * SIZE_PAGE + sizeof(map_entry_t)); @@ -1634,7 +1778,7 @@ int _map_init(vm_map_t *kmap, vm_object_t *kernel, void **bss, void **top) LIB_ASSERT_ALWAYS(result >= 0, "vm: Problem with extending kernel heap for map_entry_t pool (vaddr=%p)", *bss); } - map_common.entries = (*bss); + map_common.entries = (void *)(((ptr_t)(*bss) + (sizeof(u64) - 1U)) & ~(sizeof(u64) - 1U)); poolsz = min((ptr_t)(*top) - (ptr_t)(*bss), sizeof(map_entry_t) * map_common.ntotal); map_common.free = map_common.entries; diff --git a/vm/map.h b/vm/map.h index a9a9bca22..f5fd17528 100644 --- a/vm/map.h +++ b/vm/map.h @@ -25,6 +25,7 @@ #include "proc/lock.h" #include "amap.h" #include "types.h" +#include "page.h" struct _amap_t; @@ -37,6 +38,7 @@ typedef struct _vm_map_t { void *stop; rbtree_t tree; lock_t lock; + ph_map_t **phMaps; } vm_map_t; @@ -109,12 +111,15 @@ void vm_mapDump(vm_map_t *map); vm_attr_t vm_flagsToAttr(vm_flags_t flags); -int vm_mapCreate(vm_map_t *map, void *start, void *stop); +int vm_mapCreate(vm_map_t *map, void *start, void *stop, ph_map_t **phMaps); int vm_mapCopy(struct _process_t *proc, vm_map_t *dst, vm_map_t *src); +ph_map_t **vm_createPhMapList(const unsigned char *maps, size_t mapSz); + + void vm_mapDestroy(struct _process_t *p, vm_map_t *map); @@ -124,7 +129,7 @@ void vm_mapGetStats(size_t *allocsz); void vm_mapinfo(meminfo_t *info); -vm_map_t *vm_getSharedMap(int map); +ph_map_t *vm_getPhysicalMap(int map); int vm_mapBelongs(const struct _process_t *proc, const void *ptr, size_t size); diff --git a/vm/object.c b/vm/object.c index 87bb3c15a..9098c1909 100644 --- a/vm/object.c +++ b/vm/object.c @@ -56,7 +56,7 @@ static int object_cmp(rbnode_t *n1, rbnode_t *n2) } -int vm_objectGet(vm_object_t **o, oid_t oid) +int vm_objectGet(vm_object_t **o, oid_t oid, partition_t *part) { vm_object_t t, *no = NULL; size_t i, n; @@ -102,6 +102,7 @@ int vm_objectGet(vm_object_t **o, oid_t oid) /* Safe to cast - sz fits into size_t from above checks */ (*o)->size = (size_t)sz; (*o)->refs = 0; + (*o)->part = part; for (i = 0; i < n; ++i) { (*o)->pages[i] = NULL; @@ -155,12 +156,12 @@ int vm_objectPut(vm_object_t *o) /* Contiguous object 'holds' all pages in pages[0] */ if ((o->oid.port == (u32)(-1)) && (o->oid.id == (id_t)(-1))) { - vm_pageFree(o->pages[0]); + vm_pageFree(o->pages[0], o->part); } else { for (i = 0; i < round_page(o->size) / SIZE_PAGE; ++i) { if (o->pages[i] != NULL) { - vm_pageFree(o->pages[i]); + vm_pageFree(o->pages[i], o->part); } } } @@ -171,37 +172,37 @@ int vm_objectPut(vm_object_t *o) } -static page_t *object_fetch(oid_t oid, u64 offs) +static page_t *object_fetch(vm_map_t *map, vm_object_t *o, u64 offs) { page_t *p; void *v; - if (proc_open(oid, 0) < 0) { + if (proc_open(o->oid, 0) < 0) { return NULL; } - p = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); + p = vm_pageAlloc(map->phMaps, SIZE_PAGE, PAGE_OWNER_APP, o->part); if (p == NULL) { - (void)proc_close(oid, 0); + (void)proc_close(o->oid, 0); return NULL; } v = vm_mmap(object_common.kmap, NULL, p, SIZE_PAGE, PROT_WRITE | PROT_USER, object_common.kernel, 0, MAP_NONE); if (v == NULL) { - vm_pageFree(p); - (void)proc_close(oid, 0); + vm_pageFree(p, o->part); + (void)proc_close(o->oid, 0); return NULL; } - if (proc_read(oid, (off_t)offs, v, SIZE_PAGE, 0) < 0) { + if (proc_read(o->oid, (off_t)offs, v, SIZE_PAGE, 0) < 0) { (void)vm_munmap(object_common.kmap, v, SIZE_PAGE); - vm_pageFree(p); - (void)proc_close(oid, 0); + vm_pageFree(p, o->part); + (void)proc_close(o->oid, 0); return NULL; } (void)vm_munmap(object_common.kmap, v, SIZE_PAGE); - (void)proc_close(oid, 0); + (void)proc_close(o->oid, 0); return p; } @@ -212,8 +213,11 @@ int vm_objectPage(vm_map_t *map, amap_t **amap, vm_object_t *o, void *vaddr, u64 int err; if (o == NULL) { - *page = vm_pageAlloc(SIZE_PAGE, PAGE_OWNER_APP); - return (*page != NULL) ? EOK : -ENOMEM; + if (amap != NULL && *amap != NULL) { + *page = vm_pageAlloc(map->phMaps, SIZE_PAGE, PAGE_OWNER_APP, (*amap)->partition); + return (*page != NULL) ? EOK : -ENOMEM; + } + return -EINVAL; } if (o == VM_OBJ_PHYSMEM) { @@ -249,12 +253,12 @@ int vm_objectPage(vm_map_t *map, amap_t **amap, vm_object_t *o, void *vaddr, u64 (void)proc_lockClear(&map->lock); - *page = object_fetch(o->oid, offs); + *page = object_fetch(map, o, offs); err = vm_lockVerify(map, amap, o, vaddr, offs); if (err != 0) { if (*page != NULL) { - vm_pageFree(*page); + vm_pageFree(*page, o->part); } return err; @@ -265,7 +269,7 @@ int vm_objectPage(vm_map_t *map, amap_t **amap, vm_object_t *o, void *vaddr, u64 if (o->pages[offs / SIZE_PAGE] != NULL) { /* Someone loaded a page in the meantime, use it */ if (*page != NULL) { - vm_pageFree(*page); + vm_pageFree(*page, o->part); } *page = o->pages[offs / SIZE_PAGE]; @@ -284,8 +288,11 @@ vm_object_t *vm_objectContiguous(size_t size) vm_object_t *o; page_t *p; size_t i, n; + process_t *proc = proc_current()->process; + ph_map_t **phMaps = (proc != NULL) ? proc->mapp->phMaps : object_common.kmap->phMaps; + partition_t *part = (proc != NULL) ? proc->partition : NULL; - p = vm_pageAlloc(size, PAGE_OWNER_APP); + p = vm_pageAlloc(phMaps, size, PAGE_OWNER_APP, part); if (p == NULL) { return NULL; } @@ -295,7 +302,7 @@ vm_object_t *vm_objectContiguous(size_t size) o = vm_kmalloc(sizeof(vm_object_t) + n * sizeof(page_t *)); if (o == NULL) { - vm_pageFree(p); + vm_pageFree(p, part); return NULL; } @@ -305,6 +312,7 @@ vm_object_t *vm_objectContiguous(size_t size) o->oid.id = (id_t)(-1); o->refs = 1; o->size = size; + o->part = part; for (i = 0; i < n; ++i) { o->pages[i] = p + i; @@ -331,7 +339,7 @@ int _object_init(vm_map_t *kmap, vm_object_t *kernel) kernel->oid.id = 0; (void)lib_rbInsert(&object_common.tree, &kernel->linkage); - (void)vm_objectGet(&o, kernel->oid); + (void)vm_objectGet(&o, kernel->oid, NULL); return EOK; } diff --git a/vm/object.h b/vm/object.h index c2df15af0..f5aee9a9a 100644 --- a/vm/object.h +++ b/vm/object.h @@ -23,11 +23,13 @@ struct _vm_map_t; +struct _partition_t; typedef struct _vm_object_t { rbnode_t linkage; oid_t oid; int refs; + struct _partition_t *part; size_t size; page_t *pages[]; } vm_object_t; @@ -39,7 +41,7 @@ typedef struct _vm_object_t { vm_object_t *vm_objectRef(vm_object_t *o); -int vm_objectGet(vm_object_t **o, oid_t oid); +int vm_objectGet(vm_object_t **o, oid_t oid, struct _partition_t *part); int vm_objectPut(vm_object_t *o); diff --git a/vm/page-nommu.c b/vm/page-nommu.c index 836ec92cc..8604f9764 100644 --- a/vm/page-nommu.c +++ b/vm/page-nommu.c @@ -63,7 +63,7 @@ static page_t *_page_alloc(size_t size, vm_flags_t flags) } -page_t *vm_pageAlloc(size_t size, vm_flags_t flags) +page_t *vm_pageAlloc(ph_map_t **maps, size_t size, vm_flags_t flags, partition_t *part) { page_t *p; @@ -75,7 +75,7 @@ page_t *vm_pageAlloc(size_t size, vm_flags_t flags) } -void vm_pageFree(page_t *p) +void vm_pageFree(page_t *p, partition_t *part) { (void)proc_lockSet(&pages.lock); @@ -140,7 +140,7 @@ int _page_sbrk(pmap_t *pmap, void **start, void **end) } -void vm_pageGetStats(size_t *freesz) +void _vm_pageGetStats(size_t *freesz) { *freesz = pages.freesz; } @@ -151,22 +151,25 @@ void vm_pageinfo(meminfo_t *info) (void)proc_lockSet(&pages.lock); info->page.alloc = pages.allocsz; - info->page.free = pages.freesz; + info->page.total = pages.freesz - pages.allocsz; info->page.boot = pages.bootsz; info->page.sz = sizeof(page_t); info->page.mapsz = -1; + info->page.map.mapsz = -1; (void)proc_lockClear(&pages.lock); } -void _page_init(pmap_t *pmap, void **bss, void **top) +void _page_init(vm_map_t *kmap, void **bss, void **top) { page_t *p; const syspage_map_t *map; (void)proc_lockInit(&pages.lock, &proc_lockAttrDefault, "page.nommu"); + kmap->phMaps = NULL; + /* TODO: handle error */ map = syspage_mapAddrResolve((addr_t)&__bss_start); if (map == NULL) { diff --git a/vm/page.c b/vm/page.c index 15c2f0984..13f23b510 100644 --- a/vm/page.c +++ b/vm/page.c @@ -26,7 +26,7 @@ #define SIZE_VM_SIZES ((unsigned int)(sizeof(void *) * (size_t)__CHAR_BIT__)) -static struct { +struct _ph_map_t { page_t *sizes[SIZE_VM_SIZES]; page_t *pages; /* pages ordering and their addresses (page_t::addr) stay invariant after _page_init() */ @@ -35,10 +35,28 @@ static struct { size_t bootsz; lock_t lock; -} pages_info; + + addr_t start; + addr_t stop; +}; + +static struct { + ph_map_t **maps; + size_t mapssz; + lock_t lock; +} page_common; + + +ph_map_t *vm_getPhysicalMap(int map) +{ + if ((unsigned int)map >= page_common.mapssz) { + return NULL; + } + return page_common.maps[map]; +} -static page_t *_page_alloc(size_t size, vm_flags_t flags) +static page_t *_page_alloc(ph_map_t *map, size_t size, vm_flags_t flags, partition_t *part) { unsigned int start, stop, i; page_t *lh, *rh; @@ -52,65 +70,126 @@ static page_t *_page_alloc(size_t size, vm_flags_t flags) start++; } + if (part != NULL) { + (void)proc_lockSet(&part->lock); + if ((part->config->availableMem < part->usedMem) || + ((1UL << start) > part->config->availableMem - part->usedMem)) { + (void)proc_lockClear(&part->lock); + return NULL; + } + } + /* Find segment */ stop = start; - while ((stop < SIZE_VM_SIZES) && (pages_info.sizes[stop] == NULL)) { + while ((stop < SIZE_VM_SIZES) && (map->sizes[stop] == NULL)) { stop++; } if (stop == SIZE_VM_SIZES) { + if (part != NULL) { + (void)proc_lockClear(&part->lock); + } return NULL; } - lh = pages_info.sizes[stop]; + lh = map->sizes[stop]; /* Split segment */ while (stop > start) { - LIST_REMOVE(&pages_info.sizes[stop], lh); + LIST_REMOVE(&map->sizes[stop], lh); stop--; lh->idx--; rh = lh + (1UL << lh->idx) / SIZE_PAGE; rh->idx = lh->idx; - LIST_ADD(&pages_info.sizes[stop], lh); - LIST_ADD(&pages_info.sizes[stop], rh); + LIST_ADD(&map->sizes[stop], lh); + LIST_ADD(&map->sizes[stop], rh); } - LIST_REMOVE(&pages_info.sizes[stop], lh); + LIST_REMOVE(&map->sizes[stop], lh); /* Mark allocated pages */ for (i = 0; i < (1UL << lh->idx) / SIZE_PAGE; i++) { (lh + i)->flags &= ~PAGE_FREE; (lh + i)->flags |= flags; - pages_info.allocsz += SIZE_PAGE; + map->allocsz += SIZE_PAGE; + } + + if (part != NULL) { + part->usedMem += (1UL << lh->idx); + (void)proc_lockClear(&part->lock); } return lh; } -page_t *vm_pageAlloc(size_t size, vm_flags_t flags) +static page_t *page_allocAllMaps(size_t size, vm_flags_t flags, partition_t *part) { - page_t *p; + page_t *p = NULL; + unsigned int i; + + for (i = 0; i < page_common.mapssz; i++) { + if (page_common.maps[i] == NULL) { + continue; + } + (void)proc_lockSet(&page_common.maps[i]->lock); + p = _page_alloc(page_common.maps[i], size, flags, part); + (void)proc_lockClear(&page_common.maps[i]->lock); + if (p != NULL) { + return p; + } + } + return NULL; +} - (void)proc_lockSet(&pages_info.lock); - p = _page_alloc(size, flags); - (void)proc_lockClear(&pages_info.lock); + +page_t *vm_pageAlloc(ph_map_t **maps, size_t size, vm_flags_t flags, partition_t *part) +{ + page_t *p = NULL; + + while (*maps != NULL) { + (void)proc_lockSet(&(*maps)->lock); + p = _page_alloc(*maps, size, flags, part); + (void)proc_lockClear(&(*maps)->lock); + if (p != NULL) { + return p; + } + maps++; + } return p; } -void vm_pageFree(page_t *p) +static ph_map_t *page_mapByAddr(addr_t addr) +{ + unsigned int i; + for (i = 0; i < page_common.mapssz; i++) { + if ((page_common.maps[i]->start <= addr) && (page_common.maps[i]->stop > addr)) { + return page_common.maps[i]; + } + } + return NULL; +} + + +void vm_pageFree(page_t *p, partition_t *part) { unsigned int idx, i; page_t *lh = p, *rh = p; + ph_map_t *map; if (p == NULL) { return; } - (void)proc_lockSet(&pages_info.lock); + map = page_mapByAddr(p->addr); + if (map == NULL) { + return; + } + + (void)proc_lockSet(&map->lock); if ((lh->flags & PAGE_FREE) != 0U) { hal_cpuDisableInterrupts(); @@ -122,10 +201,17 @@ void vm_pageFree(page_t *p) idx = p->idx; + if (part != NULL) { + (void)proc_lockSet(&part->lock); + LIB_ASSERT_ALWAYS(part->usedMem >= (1UL << idx), "partition invalid free page.c"); + part->usedMem -= (1UL << idx); + (void)proc_lockClear(&part->lock); + } + /* Mark free pages */ for (i = 0; i < ((u64)1 << idx) / SIZE_PAGE; i++) { (p + i)->flags |= PAGE_FREE; - pages_info.allocsz -= SIZE_PAGE; + map->allocsz -= SIZE_PAGE; } if ((p->addr & (((u64)1 << (idx + 1U)) - 1U)) != 0U) { @@ -136,15 +222,15 @@ void vm_pageFree(page_t *p) } /* parasoft-suppress-next-line MISRAC2012-DIR_4_1 MISRAC2012-RULE_18_3 "lh, rh, pages_info.pages are related" */ - while ((lh >= pages_info.pages) && (rh < (pages_info.pages + pages_info.totalsz / SIZE_PAGE)) && + while ((lh >= map->pages) && (rh < (map->pages + map->totalsz / SIZE_PAGE)) && ((lh->flags & PAGE_FREE) != 0U) && ((rh->flags & PAGE_FREE) != 0U) && (lh->idx == rh->idx) && ((lh->addr + (1UL << lh->idx)) == rh->addr) && (idx < SIZE_VM_SIZES)) { if (p == lh) { - LIST_REMOVE(&pages_info.sizes[idx], rh); + LIST_REMOVE(&map->sizes[idx], rh); } else { - LIST_REMOVE(&pages_info.sizes[idx], lh); + LIST_REMOVE(&map->sizes[idx], lh); } rh->idx = (u8)hal_cpuGetFirstBit(SIZE_PAGE); @@ -161,9 +247,9 @@ void vm_pageFree(page_t *p) } } - LIST_ADD(&pages_info.sizes[idx], p); + LIST_ADD(&map->sizes[idx], p); - (void)proc_lockClear(&pages_info.lock); + (void)proc_lockClear(&map->lock); return; } @@ -188,26 +274,30 @@ static int page_get_cmp(void *key, void *item) page_t *page_get(addr_t addr) { page_t *p; - size_t np = pages_info.totalsz / SIZE_PAGE; + ph_map_t *map = page_mapByAddr(addr); + if (map == NULL) { + return NULL; + } + size_t np = map->totalsz / SIZE_PAGE; addr = addr & ~(SIZE_PAGE - 1U); - p = lib_bsearch((void *)addr, pages_info.pages, np, sizeof(page_t), page_get_cmp); + p = lib_bsearch((void *)addr, map->pages, np, sizeof(page_t), page_get_cmp); return p; } -static void _page_initSizes(void) +static void _page_initSizes(ph_map_t *map) { unsigned int idx; size_t k, i = 0; page_t *p; /* Remove already discovered pages */ - pages_info.sizes[hal_cpuGetFirstBit(SIZE_PAGE)] = NULL; + map->sizes[hal_cpuGetFirstBit(SIZE_PAGE)] = NULL; - while (i < pages_info.totalsz / SIZE_PAGE) { - p = &pages_info.pages[i]; + while (i < map->totalsz / SIZE_PAGE) { + p = &map->pages[i]; if ((p->flags & PAGE_FREE) == 0U) { i++; continue; @@ -220,8 +310,8 @@ static void _page_initSizes(void) } /* parasoft-suppress-next-line MISRAC2012-DIR_4_1 "idx is limited to min(SIZE_VM_SIZES - 1U, bits in p-> addr")*/ - for (k = 0U; (k < (((u64)1 << idx) / SIZE_PAGE) - 1U) && (i + k < ((pages_info.totalsz / SIZE_PAGE) - 1U)); k++) { - if ((pages_info.pages[i + k + 1U].flags & PAGE_FREE) == 0U) { + for (k = 0U; (k < (((u64)1 << idx) / SIZE_PAGE) - 1U) && (i + k < ((map->totalsz / SIZE_PAGE) - 1U)); k++) { + if ((map->pages[i + k + 1U].flags & PAGE_FREE) == 0U) { break; } } @@ -229,7 +319,7 @@ static void _page_initSizes(void) idx = hal_cpuGetLastBit((k + 1U) * SIZE_PAGE); p->idx = (u8)idx; - LIST_ADD(&pages_info.sizes[idx], p); + LIST_ADD(&map->sizes[idx], p); i += (size_t)(((u64)1 << idx) / SIZE_PAGE); } @@ -251,18 +341,17 @@ static unsigned int page_digits(unsigned int n, unsigned int base) #define TTY_COLS 80U -void _page_showPages(void) +static void _page_showMapPages(ph_map_t *map) { addr_t a = 0; page_t *p; unsigned int rep, i = 0, k; - int w; + int w = 0; char c; char buf[TTY_COLS + 1U]; - w = lib_sprintf(buf, "vm: "); - while (i < pages_info.totalsz / SIZE_PAGE) { - p = &pages_info.pages[i]; + while (i < map->totalsz / SIZE_PAGE) { + p = &map->pages[i]; /* Print markers in case of memory gap */ if (p->addr > a) { @@ -288,8 +377,8 @@ void _page_showPages(void) /* Print markers with repetitions */ c = pmap_marker(p); - for (rep = 0; ((size_t)i + rep + 1U) < pages_info.totalsz / SIZE_PAGE; rep++) { - if ((c != pmap_marker(&pages_info.pages[i + rep + 1U])) || (pages_info.pages[i + rep + 1U].addr - pages_info.pages[i + rep].addr > SIZE_PAGE)) { + for (rep = 0; ((size_t)i + rep + 1U) < map->totalsz / SIZE_PAGE; rep++) { + if ((c != pmap_marker(&map->pages[i + rep + 1U])) || (map->pages[i + rep + 1U].addr - map->pages[i + rep].addr > SIZE_PAGE)) { break; } } @@ -312,24 +401,33 @@ void _page_showPages(void) } } - a = pages_info.pages[i + rep].addr + SIZE_PAGE; + a = map->pages[i + rep].addr + SIZE_PAGE; i += rep + 1U; } - if (w > 4) { - lib_printf("%s\n", buf); - } + lib_printf("%s\n", buf); return; } +void _page_showPages(void) +{ + unsigned int map; + + for (map = 0; map < page_common.mapssz; map++) { + lib_printf("vm: Map 0x%x:0x%x ", (void *)page_common.maps[map]->start, (void *)page_common.maps[map]->stop); + _page_showMapPages(page_common.maps[map]); + } +} + + static int _page_map(pmap_t *pmap, void *vaddr, addr_t pa, vm_attr_t attr) { page_t *ap = NULL; while (pmap_enter(pmap, pa, vaddr, attr, ap) < 0) { - ap = _page_alloc(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE); + ap = page_allocAllMaps(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE, NULL); if (/*vaddr > (void *)VADDR_KERNEL ||*/ ap == NULL) { return -ENOMEM; } @@ -342,9 +440,9 @@ int page_map(pmap_t *pmap, void *vaddr, addr_t pa, vm_attr_t attr) { int err; - (void)proc_lockSet(&pages_info.lock); + (void)proc_lockSet(&page_common.lock); err = _page_map(pmap, vaddr, pa, attr); - (void)proc_lockClear(&pages_info.lock); + (void)proc_lockClear(&page_common.lock); return err; } @@ -353,13 +451,13 @@ int page_map(pmap_t *pmap, void *vaddr, addr_t pa, vm_attr_t attr) int _page_sbrk(pmap_t *pmap, void **start, void **end) { page_t *np, *ap = NULL; - np = _page_alloc(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP); + np = page_allocAllMaps(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP, NULL); if (np == NULL) { return -ENOMEM; } while (pmap_enter(pmap, np->addr, (*end), PGHD_READ | PGHD_WRITE | PGHD_PRESENT, ap) < 0) { - ap = _page_alloc(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE); + ap = page_allocAllMaps(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE, NULL); if (ap == NULL) { return -ENOMEM; } @@ -371,9 +469,12 @@ int _page_sbrk(pmap_t *pmap, void **start, void **end) } -void vm_pageGetStats(size_t *freesz) +void _vm_pageGetStats(size_t *freesz) { - *freesz = pages_info.totalsz - pages_info.allocsz; + *freesz = 0; + for (size_t i = 0; i < page_common.mapssz; i++) { + *freesz += page_common.maps[i]->totalsz - page_common.maps[i]->allocsz; + } } @@ -381,72 +482,93 @@ void vm_pageinfo(meminfo_t *info) { char c; page_t *p; + int idx; unsigned int rep, i = 0; int size = 0; - (void)proc_lockSet(&pages_info.lock); - - info->page.alloc = (unsigned int)pages_info.allocsz; - info->page.free = (unsigned int)(pages_info.totalsz - pages_info.allocsz); - info->page.boot = (unsigned int)pages_info.bootsz; info->page.sz = (unsigned int)sizeof(page_t); + info->page.mapsz = (int)page_common.mapssz; + info->page.alloc = 0; + info->page.total = 0; + info->page.boot = 0; + for (i = 0; i < page_common.mapssz; i++) { + (void)proc_lockSet(&page_common.maps[i]->lock); + info->page.alloc += (unsigned int)page_common.maps[i]->allocsz; + info->page.total += (unsigned int)page_common.maps[i]->totalsz; + info->page.boot += (unsigned int)page_common.maps[i]->bootsz; + (void)proc_lockClear(&page_common.maps[i]->lock); + } - if (info->page.mapsz != -1) { - while (i < pages_info.totalsz / SIZE_PAGE) { - p = pages_info.pages + i; + idx = info->page.mapidx; + if ((idx == -1) || (idx >= (int)page_common.mapssz)) { + return; + } + + (void)proc_lockSet(&page_common.maps[idx]->lock); + + info->page.map.alloc = (unsigned int)page_common.maps[idx]->allocsz; + info->page.map.free = (unsigned int)(page_common.maps[idx]->totalsz - page_common.maps[idx]->allocsz); + info->page.map.boot = (unsigned int)page_common.maps[idx]->bootsz; + + if (info->page.map.mapsz != -1) { + i = 0; + while (i < page_common.maps[idx]->totalsz / SIZE_PAGE) { + p = page_common.maps[idx]->pages + i; c = pmap_marker(p); - for (rep = 0; ((size_t)i + rep + 1U) < pages_info.totalsz / SIZE_PAGE; rep++) { - if ((c != pmap_marker(pages_info.pages + i + rep + 1U)) || ((pages_info.pages[i + rep + 1U].addr - pages_info.pages[i + rep].addr) > SIZE_PAGE)) { + for (rep = 0; ((size_t)i + rep + 1U) < page_common.maps[idx]->totalsz / SIZE_PAGE; rep++) { + if ((c != pmap_marker(page_common.maps[idx]->pages + i + rep + 1U)) || ((page_common.maps[idx]->pages[i + rep + 1U].addr - page_common.maps[idx]->pages[i + rep].addr) > SIZE_PAGE)) { break; } } - if (info->page.mapsz > size && info->page.map != NULL) { - info->page.map[size].count = rep + 1U; - info->page.map[size].marker = c; - info->page.map[size].addr = p->addr; + if (info->page.map.mapsz > size && info->page.map.map != NULL) { + info->page.map.map[size].count = rep + 1U; + info->page.map.map[size].marker = c; + info->page.map.map[size].addr = p->addr; } i += rep + 1U; ++size; } - info->page.mapsz = size; + info->page.map.mapsz = size; } - (void)proc_lockClear(&pages_info.lock); + (void)proc_lockClear(&page_common.maps[idx]->lock); } -void _page_init(pmap_t *pmap, void **bss, void **top) +static int _page_initMap(ph_map_t *map, const syspage_map_t *sysMap, pmap_t *pmap, void **bss, void **top) { addr_t addr; unsigned int k; - page_t *page, *p; + page_t *page; int err; - void *vaddr; - (void)proc_lockInit(&pages_info.lock, &proc_lockAttrDefault, "page"); + map->start = sysMap->start; + map->stop = sysMap->end; + + (void)proc_lockInit(&map->lock, &proc_lockAttrDefault, "page.map"); /* Prepare memory hash */ - pages_info.totalsz = 0; - pages_info.allocsz = 0; - pages_info.bootsz = 0; + map->totalsz = 0; + map->allocsz = 0; + map->bootsz = 0; for (k = 0; k < SIZE_VM_SIZES; k++) { - pages_info.sizes[k] = NULL; + map->sizes[k] = NULL; } - addr = 0; - pages_info.pages = (page_t *)*bss; + addr = sysMap->start; + map->pages = (page_t *)*bss; page = (page_t *)*bss; for (;;) { if ((void *)page + sizeof(page_t) >= (*top)) { if (_page_sbrk(pmap, bss, top) < 0) { lib_printf("vm: Kernel heap extension error %p %p!\n", page, *top); - return; + return -ENOMEM; } } @@ -456,23 +578,23 @@ void _page_init(pmap_t *pmap, void **bss, void **top) } if (err == EOK) { - pages_info.totalsz += SIZE_PAGE; + map->totalsz += SIZE_PAGE; if ((page->flags & PAGE_FREE) != 0U) { page->idx = (u8)hal_cpuGetFirstBit(SIZE_PAGE); - LIST_ADD(&pages_info.sizes[hal_cpuGetFirstBit(SIZE_PAGE)], page); + LIST_ADD(&map->sizes[hal_cpuGetFirstBit(SIZE_PAGE)], page); } else { page->idx = 0; - pages_info.allocsz += SIZE_PAGE; + map->allocsz += SIZE_PAGE; if (((page->flags >> 1U) & 7U) == PAGE_OWNER_BOOT) { - pages_info.bootsz += SIZE_PAGE; + map->bootsz += SIZE_PAGE; } } page = page + 1; } - /* Wrap over 0 */ - if (addr < SIZE_PAGE) { + /* Skip other maps and don't wrap over 0 */ + if ((addr >= sysMap->end) || (addr < SIZE_PAGE)) { break; } } @@ -480,30 +602,78 @@ void _page_init(pmap_t *pmap, void **bss, void **top) (*bss) = page; /* Prepare allocation hash */ - _page_initSizes(); + _page_initSizes(map); + + return 0; +} + + +void _page_init(vm_map_t *kmap, void **bss, void **top) +{ + meminfo_t info; + page_t *p; + void *vaddr; + const syspage_map_t *sysMap = syspage_mapList(); + size_t size; + unsigned int i = 0; + (void)proc_lockInit(&page_common.lock, &proc_lockAttrDefault, "page.common"); + + page_common.maps = (ph_map_t **)(*bss); + page_common.mapssz = syspage_mapSize(); + size = sizeof(ph_map_t *) * (page_common.mapssz + 1U); + if ((ptr_t)page_common.maps + size >= (ptr_t)(*top)) { + if (_page_sbrk(&kmap->pmap, bss, top) < 0) { + lib_printf("vm: Kernel heap extension error %p %p!\n", page_common.maps, *top); + return; + } + } + *bss += size; + hal_memset(page_common.maps, 0, size); + + /* Initialize maps */ + do { + page_common.maps[i] = (ph_map_t *)(*bss); + if ((ptr_t)page_common.maps[i] + sizeof(ph_map_t) >= (ptr_t)(*top)) { + if (_page_sbrk(&kmap->pmap, bss, top) < 0) { + lib_printf("vm: Kernel heap extension error %p %p!\n", page_common.maps[i], *top); + return; + } + } + *bss += sizeof(ph_map_t); + if (_page_initMap(page_common.maps[i], sysMap, &kmap->pmap, bss, top) != 0) { + lib_printf("vm: Error during physical map initialization!\n"); + return; + } + + i++; + sysMap = sysMap->next; + } while (sysMap != syspage_mapList()); + + kmap->phMaps = page_common.maps; + + info.page.mapidx = -1; + vm_pageinfo(&info); /* Initialize kernel space for user processes */ p = NULL; vaddr = (*top); for (;;) { - if (_pmap_kernelSpaceExpand(pmap, &vaddr, (*top) + max(pages_info.totalsz / 4U, (1UL << 23)), p) == 0) { + if (_pmap_kernelSpaceExpand(&kmap->pmap, &vaddr, (*top) + max(info.page.total / 4U, (1UL << 23)), p) == 0) { break; } - p = _page_alloc(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE); + p = page_allocAllMaps(SIZE_PAGE, PAGE_OWNER_KERNEL | PAGE_KERNEL_PTABLE, NULL); if (p == NULL) { return; } } /* Show statistics on the console */ - lib_printf("vm: Initializing page allocator (%d+%d)/%dKB, page_t=%d\n", (pages_info.allocsz - pages_info.bootsz) / 1024U, - pages_info.bootsz / 1024U, pages_info.totalsz / 1024U, sizeof(page_t)); + lib_printf("vm: Initializing page allocator (%d+%d)/%dKB, page_t=%d\n", (info.page.alloc - info.page.boot) / 1024U, + info.page.boot / 1024U, (info.page.total) / 1024U, sizeof(page_t)); _page_showPages(); /* Create NULL pointer entry */ - (void)_page_map(pmap, NULL, 0, PGHD_USER | ~PGHD_PRESENT); - - return; + (void)page_map(&kmap->pmap, NULL, 0, PGHD_USER | ~PGHD_PRESENT); } diff --git a/vm/page.h b/vm/page.h index 77cdeb865..8da2d702e 100644 --- a/vm/page.h +++ b/vm/page.h @@ -22,10 +22,22 @@ #include "types.h" -page_t *vm_pageAlloc(size_t size, vm_flags_t flags); +struct _partition_t; +struct _vm_map_t; -void vm_pageFree(page_t *p); +#ifndef NOMMU +struct _ph_map_t; +typedef struct _ph_map_t ph_map_t; +#else +typedef struct _vm_map_t ph_map_t; +#endif + + +page_t *vm_pageAlloc(ph_map_t **maps, size_t size, vm_flags_t flags, struct _partition_t *part); + + +void vm_pageFree(page_t *p, struct _partition_t *part); /* returns NULL when addr is outside of defined physical maps (MMU) */ @@ -41,13 +53,13 @@ int page_map(pmap_t *pmap, void *vaddr, addr_t pa, vm_attr_t attr); int _page_sbrk(pmap_t *pmap, void **start, void **end); -void vm_pageGetStats(size_t *freesz); +void _vm_pageGetStats(size_t *freesz); void vm_pageinfo(meminfo_t *info); -void _page_init(pmap_t *pmap, void **bss, void **top); +void _page_init(struct _vm_map_t *kmap, void **bss, void **top); #endif diff --git a/vm/vm.c b/vm/vm.c index 113094519..5ed9a41de 100644 --- a/vm/vm.c +++ b/vm/vm.c @@ -42,7 +42,7 @@ void _vm_init(vm_map_t *kmap, vm_object_t *kernel) { /* parasoft-suppress-next-line MISRAC2012-RULE_18_1 "&vm.top is passed as a reference to vm.top not as an array object" */ _pmap_init(&kmap->pmap, &vm.bss, &vm.top); - _page_init(&kmap->pmap, &vm.bss, &vm.top); + _page_init(kmap, &vm.bss, &vm.top); (void)_map_init(kmap, kernel, &vm.bss, &vm.top); diff --git a/vm/zone.c b/vm/zone.c index 37ab7fbed..ca00e9e9f 100644 --- a/vm/zone.c +++ b/vm/zone.c @@ -37,14 +37,14 @@ int _vm_zoneCreate(vm_zone_t *zone, size_t blocksz, unsigned int blocks) return -EINVAL; } - zone->pages = vm_pageAlloc(blocks * blocksz, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP); + zone->pages = vm_pageAlloc(zone_common.kmap->phMaps, blocks * blocksz, PAGE_OWNER_KERNEL | PAGE_KERNEL_HEAP, NULL); if (zone->pages == NULL) { return -ENOMEM; } zone->vaddr = vm_mmap(zone_common.kmap, zone_common.kmap->start, zone->pages, (size_t)1U << zone->pages->idx, PROT_READ | PROT_WRITE, zone_common.kernel, -1, MAP_NONE); if (zone->vaddr == NULL) { - vm_pageFree(zone->pages); + vm_pageFree(zone->pages, NULL); return -ENOMEM; } @@ -74,7 +74,7 @@ int _vm_zoneDestroy(vm_zone_t *zone) } (void)vm_munmap(zone_common.kmap, zone->vaddr, (size_t)1U << zone->pages->idx); - vm_pageFree(zone->pages); + vm_pageFree(zone->pages, NULL); zone->vaddr = NULL; zone->first = NULL;