diff --git a/.config-default b/.config-default index 53985115..6616ddbf 100644 --- a/.config-default +++ b/.config-default @@ -49,6 +49,8 @@ CONFIG_SCHED_WAKEUP_GRANULARITY=0 CONFIG_SCHED_LOAD_BALANCE_INTERVAL=8 CONFIG_SCHED_BALANCE_BATCH=4 CONFIG_SCHED_MIGRATION_COOLDOWN=4 +CONFIG_SCHED_STEAL_HOT_BUDGET=8 +CONFIG_SCHED_STEAL_SCAN_LIMIT=32 CONFIG_TASK_DEFAULT_SLICE=5 CONFIG_SCHED_DOMAIN_MAX_LEVELS=3 diff --git a/Kconfig b/Kconfig index 26232452..b6c331eb 100644 --- a/Kconfig +++ b/Kconfig @@ -188,6 +188,26 @@ menu "Scheduler" Hysteresis against repeatedly bouncing a hot task between CPUs. Idle-CPU stealing may override this when leaving a CPU idle would be worse. + config SCHED_STEAL_HOT_BUDGET + int "Recently migrated candidates skipped per steal attempt" + default 8 + range 0 64 + help + When a steal scan is not running for a newly idle CPU and the source + runqueue has at most two runnable tasks, candidates migrated within + SCHED_MIGRATION_COOLDOWN are skipped to avoid bouncing hot tasks. This + bounds how many such candidates one attempt passes over before + abandoning the source runqueue. Zero stops at the first hot candidate. + + config SCHED_STEAL_SCAN_LIMIT + int "Maximum runqueue nodes visited per steal attempt" + default 32 + range 1 256 + help + Hard bound on rb-tree nodes examined by one steal attempt, including + CPU-pinned entities. Keeps load-balancing cost bounded on large + runqueues. Must be at least SCHED_STEAL_HOT_BUDGET. + config TASK_DEFAULT_SLICE int "Default initial task time slice (ticks)" default 5 diff --git a/drivers/firmware/apic.c b/drivers/firmware/apic.c index 883f684b..bf325cfb 100644 --- a/drivers/firmware/apic.c +++ b/drivers/firmware/apic.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -313,7 +314,7 @@ void send_ipi(uint32_t apic_id, uint32_t command) break; } } - if (rflags & (1ULL << 9)) enable_intr(); + if (rflags & X86_EFLAGS_IF) enable_intr(); } /* Initialize APIC */ diff --git a/include/arch/cpu/eflags.h b/include/arch/cpu/eflags.h new file mode 100644 index 00000000..46c23b4e --- /dev/null +++ b/include/arch/cpu/eflags.h @@ -0,0 +1,34 @@ +/* + * + * eflags.h + * x86-64 RFLAGS bits + * + * 2026/10/23 By MicroFish + * Copyright (C) 2020 ViudiraTech, based on the Apache 2.0 license. + * + */ + +#ifndef INCLUDE_EFLAGS_H_ +#define INCLUDE_EFLAGS_H_ + +/* RFLAGS bits */ +#define X86_EFLAGS_CF (1ULL << 0) +#define X86_EFLAGS_FIXED (1ULL << 1) +#define X86_EFLAGS_PF (1ULL << 2) +#define X86_EFLAGS_AF (1ULL << 4) +#define X86_EFLAGS_ZF (1ULL << 6) +#define X86_EFLAGS_SF (1ULL << 7) +#define X86_EFLAGS_TF (1ULL << 8) +#define X86_EFLAGS_IF (1ULL << 9) +#define X86_EFLAGS_DF (1ULL << 10) +#define X86_EFLAGS_OF (1ULL << 11) +#define X86_EFLAGS_IOPL (3ULL << 12) +#define X86_EFLAGS_NT (1ULL << 14) +#define X86_EFLAGS_RF (1ULL << 16) +#define X86_EFLAGS_VM (1ULL << 17) +#define X86_EFLAGS_AC (1ULL << 18) +#define X86_EFLAGS_VIF (1ULL << 19) +#define X86_EFLAGS_VIP (1ULL << 20) +#define X86_EFLAGS_ID (1ULL << 21) + +#endif // INCLUDE_EFLAGS_H_ diff --git a/include/kernel/config.h b/include/kernel/config.h index 664be537..84e7f12c 100644 --- a/include/kernel/config.h +++ b/include/kernel/config.h @@ -357,6 +357,14 @@ # define CONFIG_SCHED_MIGRATION_COOLDOWN 4 #endif +#ifndef CONFIG_SCHED_STEAL_HOT_BUDGET +# define CONFIG_SCHED_STEAL_HOT_BUDGET 8 +#endif + +#ifndef CONFIG_SCHED_STEAL_SCAN_LIMIT +# define CONFIG_SCHED_STEAL_SCAN_LIMIT 32 +#endif + #ifndef CONFIG_SCHED_LOAD_BALANCE_INTERVAL # define CONFIG_SCHED_LOAD_BALANCE_INTERVAL 8 #endif diff --git a/include/process/sched.h b/include/process/sched.h index 9511a0b2..1776508e 100644 --- a/include/process/sched.h +++ b/include/process/sched.h @@ -15,6 +15,8 @@ #include #include +_Static_assert(CONFIG_SCHED_STEAL_HOT_BUDGET <= CONFIG_SCHED_STEAL_SCAN_LIMIT, "steal hot budget exceeds scan limit"); + #define SCHED_NICE_0_LOAD 1024ULL // Weight of a nice-0 task (EEVDF) typedef enum { diff --git a/kernel/process/elf_loader.c b/kernel/process/elf_loader.c index 5f8e828a..ec900f20 100644 --- a/kernel/process/elf_loader.c +++ b/kernel/process/elf_loader.c @@ -9,6 +9,7 @@ */ #include +#include #include #include #include @@ -720,7 +721,7 @@ int elf_loader_load_process_internal(process_t *proc, const uint8_t *elf_data, s proc->task->context.r13 = 0; proc->task->context.r14 = 0; proc->task->context.r15 = 0; - proc->task->context.rflags = 0x202; + proc->task->context.rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; proc->task->context.rdi = 0; if (!entry_out && !rsp_out) { @@ -729,7 +730,7 @@ int elf_loader_load_process_internal(process_t *proc, const uint8_t *elf_data, s *(--kstack) = 0x23; *(--kstack) = user_rsp; - *(--kstack) = 0x202; + *(--kstack) = X86_EFLAGS_IF | X86_EFLAGS_FIXED; *(--kstack) = 0x1B; *(--kstack) = actual_entry; *(--kstack) = (uint64_t)user_process_enter; @@ -875,7 +876,7 @@ int elf_loader_load_user_node(process_t *proc, vfs_node_t node, char *const argv proc->task->context.r13 = 0; proc->task->context.r14 = 0; proc->task->context.r15 = 0; - proc->task->context.rflags = 0x202; + proc->task->context.rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; proc->task->context.rdi = 0; if (entry_out) *entry_out = actual_entry; if (rsp_out) *rsp_out = user_rsp; diff --git a/kernel/process/kthread.c b/kernel/process/kthread.c index c226de10..b7e08106 100644 --- a/kernel/process/kthread.c +++ b/kernel/process/kthread.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -64,7 +65,7 @@ static int setup_kthread_stack(task_t *task, kthread_bootstrap_t *bootstrap) task->context.r13 = 0; task->context.r14 = 0; task->context.r15 = 0; - task->context.rflags = 0x202; + task->context.rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; task->context.rdi = (uint64_t)bootstrap; return 0; } @@ -139,7 +140,7 @@ void kthreadd_init(void) *(--stack) = 0; *(--stack) = (uint64_t)kthreadd_main; task->context.rsp = (uint64_t)stack; - task->context.rflags = 0x202; + task->context.rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; task->context.rdi = 0; process_t *proc = process_create_kthread(task, "kthreadd"); diff --git a/kernel/sched/sched.c b/kernel/sched/sched.c index 2621de66..50fd0aba 100644 --- a/kernel/sched/sched.c +++ b/kernel/sched/sched.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -449,7 +450,7 @@ static task_t *local_current(void) /* Restore the interrupt flag saved by get_rflags(). */ static void restore_intr(uint64_t rflags) { - if (rflags & (1ULL << 9)) + if (rflags & X86_EFLAGS_IF) enable_intr(); else disable_intr(); @@ -465,7 +466,7 @@ static void local_rq_set_curr(eevdf_rq_t *rq, task_t *task) /* Unlock keeping interrupts off; the caller restores them later. */ static void rq_unlock_irqs_off(eevdf_rq_t *rq, uint64_t rflags) { - spin_unlock_irqrestore(&rq->lock, rflags & ~(1ULL << 9)); + spin_unlock_irqrestore(&rq->lock, rflags & ~X86_EFLAGS_IF); } /* Point the TSS stack at the given task's kernel stack */ @@ -864,7 +865,9 @@ static uint64_t placement_score_locked(task_t *task, uint32_t cpu, uint32_t prev /* WF_SYNC-like hint: the waker may block/yield soon, so stacking is a bit cheaper. */ uint32_t this_cpu = get_current_cpu_id(); if (sync && cpu == this_cpu && score >= SCHED_SYNC_WAKE_BONUS) score -= SCHED_SYNC_WAKE_BONUS; - if (task && cpu != prev_cpu && scheduler.ticks - task->last_migrate_tick < CONFIG_SCHED_MIGRATION_COOLDOWN) score += SCHED_NICE_0_LOAD / 2ULL; + + uint64_t cooldown = CONFIG_SCHED_MIGRATION_COOLDOWN; + if (task && cpu != prev_cpu && cooldown && scheduler.ticks - task->last_migrate_tick < cooldown) score += SCHED_NICE_0_LOAD / 2ULL; return score; } @@ -1010,18 +1013,24 @@ static task_t *pick_steal_candidate_locked(eevdf_rq_t *src, bool newly_idle) if (!node) return NULL; while (node->right) node = node->right; - /* Pinned entities are skipped without spending the movable-candidate budget. */ - unsigned int movable = 0; - unsigned int visited = 0; - while (node && movable < 8 && visited++ < 32) { + /* Pinned entities are skipped without spending the hot-candidate budget. */ + unsigned int hot_skipped = 0; + unsigned int hot_budget = CONFIG_SCHED_STEAL_HOT_BUDGET; + unsigned int visited = 0; + uint64_t cooldown = CONFIG_SCHED_MIGRATION_COOLDOWN; + while (node && visited++ < CONFIG_SCHED_STEAL_SCAN_LIMIT) { task_t *task = rb_entry(node, task_t, run_node); + if (task->cpu_pinned) { node = rb_prev_local(node); continue; } - bool hot = scheduler.ticks - task->last_migrate_tick < CONFIG_SCHED_MIGRATION_COOLDOWN; + bool hot = cooldown && scheduler.ticks - task->last_migrate_tick < cooldown; + if (!hot || newly_idle || __atomic_load_n(&src->nr_running, __ATOMIC_RELAXED) > 2) return task; - movable++; + hot_skipped++; + + if (hot_skipped >= hot_budget) break; node = rb_prev_local(node); } return NULL; @@ -1226,7 +1235,7 @@ static task_t *idle_task_alloc(uint32_t cpu_id) *(--stack) = (uint64_t)idle_thread; idle->context.rsp = (uint64_t)stack; idle->context.rdi = 0; - idle->context.rflags = 0x202; + idle->context.rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; return idle; } diff --git a/kernel/signal/signal.c b/kernel/signal/signal.c index 07a3eac5..c95bbb86 100644 --- a/kernel/signal/signal.c +++ b/kernel/signal/signal.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -1891,8 +1892,8 @@ int64_t do_rt_sigreturn(syscall_frame_t *frame) * Clear IOPL, NT and VM so IRETQ cannot enter an invalid privilege state. */ const uint64_t user_rflags - = (1ULL << 0) | (1ULL << 2) | (1ULL << 4) | (1ULL << 6) | (1ULL << 7) | (1ULL << 8) | (1ULL << 9) | (1ULL << 10) | (1ULL << 11) | (1ULL << 16) | (1ULL << 18) | (1ULL << 21); - frame->rflags = (gregs[17] & user_rflags) | (1ULL << 1) | (1ULL << 9); + = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_TF | X86_EFLAGS_IF | X86_EFLAGS_DF | X86_EFLAGS_OF | X86_EFLAGS_RF | X86_EFLAGS_AC | X86_EFLAGS_ID; + frame->rflags = (gregs[17] & user_rflags) | X86_EFLAGS_FIXED | X86_EFLAGS_IF; frame->rsp = gregs[15]; frame->cs = 0x33; frame->ss = 0x2b; diff --git a/kernel/syscall/syscall.c b/kernel/syscall/syscall.c index 1073c295..b3f5cca3 100644 --- a/kernel/syscall/syscall.c +++ b/kernel/syscall/syscall.c @@ -8,6 +8,7 @@ * */ +#include #include #include #include @@ -74,7 +75,6 @@ _Static_assert(sizeof(syscall_frame_t) == 20 * sizeof(uint64_t), "syscall frame * flag must be clear (SMAP correctness requires IRET). */ #define X86_64_CANONICAL_BOUNDARY 0x0000800000000000ULL -#define X86_EFLAGS_AC 0x00040000ULL #define WNOHANG 0x00000001 #define WUNTRACED 0x00000002 @@ -4504,7 +4504,7 @@ static int64_t do_execve_resolved(const char *path, vfs_node_t initial_node, cha memset(frame, 0, sizeof(*frame)); frame->rip = entry; frame->cs = 0x1B; - frame->rflags = 0x202; + frame->rflags = X86_EFLAGS_IF | X86_EFLAGS_FIXED; frame->rsp = rsp; frame->ss = 0x23; } @@ -5938,7 +5938,7 @@ void syscall_init_cpu(void) star |= ((uint64_t)0x08 << 32) | ((uint64_t)0x23 << 48); wrmsr(0xC0000081, star); wrmsr(0xC0000082, (uint64_t)syscall_entry_syscall); - wrmsr(0xC0000084, 0x200); + wrmsr(0xC0000084, X86_EFLAGS_IF); wrmsr(0xC0000080, rdmsr(0xC0000080) | 1); } diff --git a/scripts/kconfig.mk b/scripts/kconfig.mk index 93c4454c..5d11b776 100644 --- a/scripts/kconfig.mk +++ b/scripts/kconfig.mk @@ -146,6 +146,14 @@ ifneq ($(CONFIG_SCHED_MIGRATION_COOLDOWN),) C_CONFIG += -DCONFIG_SCHED_MIGRATION_COOLDOWN=$(CONFIG_SCHED_MIGRATION_COOLDOWN) endif +ifneq ($(CONFIG_SCHED_STEAL_HOT_BUDGET),) + C_CONFIG += -DCONFIG_SCHED_STEAL_HOT_BUDGET=$(CONFIG_SCHED_STEAL_HOT_BUDGET) +endif + +ifneq ($(CONFIG_SCHED_STEAL_SCAN_LIMIT),) + C_CONFIG += -DCONFIG_SCHED_STEAL_SCAN_LIMIT=$(CONFIG_SCHED_STEAL_SCAN_LIMIT) +endif + ifneq ($(CONFIG_PROCESS_MAX_FD),) C_CONFIG += -DCONFIG_PROCESS_MAX_FD=$(CONFIG_PROCESS_MAX_FD) endif