Skip to content

sched: fix local current/runqueue migration races #108

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
FengHeting merged 2 commits into from
Oct 22, 2026
Merged

sched: fix local current/runqueue migration races #108

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
53e0df4
sched: resolve the local runqueue and current task with interrupts off
JiTianYu391 Oct 21, 2026
10bf701
sched: add the local preemption fast path and tighten curr updates
FengHeting Oct 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 72 additions & 29 deletions kernel/sched/sched.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -435,10 +435,37 @@ static eevdf_rq_t *local_rq(void)
return &cpu_rqs[percpu_gs_cpu_id()];
}

/* Return the task currently running on this CPU */
/*
* Return the task currently running on this CPU. This must be a single
* GS-relative load: reading the CPU number and then cpu_rqs[cpu].curr is not
* atomic against preemption, so a task migrated between the two loads would
* get whatever now runs on its old CPU (often that CPU's idle task).
*/
static task_t *local_current(void)
{
return __atomic_load_n(&local_rq()->curr, __ATOMIC_RELAXED);
return percpu_gs_current();
}

/* Restore the interrupt flag saved by get_rflags(). */
static void restore_intr(uint64_t rflags)
{
if (rflags & (1ULL << 9))
enable_intr();
else
disable_intr();
}

/* Keep rq->curr and this CPU's GS current in lockstep; rq must be local. */
static void local_rq_set_curr(eevdf_rq_t *rq, task_t *task)
{
__atomic_store_n(&rq->curr, task, __ATOMIC_RELAXED);
percpu_gs_set_current(task);
}

/* Unlock keeping interrupts off; the caller restores them later. */
static void rq_unlock_irqs_off(eevdf_rq_t *rq, uint64_t rflags)
{
spin_unlock_irqrestore(&rq->lock, rflags & ~(1ULL << 9));
}

/* Point the TSS stack at the given task's kernel stack */
Expand Down Expand Up @@ -1220,8 +1247,7 @@ void sched_init(void)
for (uint32_t i = 0; i < cpu_scheduler_count; i++) cpu_rqs[i].online = 1;
sched_domain_build();

next_task_cpu = 0;
__atomic_store_n(&cpu_rqs[0].curr, &boot_task, __ATOMIC_RELAXED);
next_task_cpu = 0;
cpu_rqs[0].nr_running = 0;

memset(&boot_task, 0, sizeof(boot_task));
Expand All @@ -1238,7 +1264,7 @@ void sched_init(void)

/* boot_task is the swapper/0 idle task for CPU 0 */
cpu_rqs[0].idle = &boot_task;
percpu_gs_set_current(&boot_task);
local_rq_set_curr(&cpu_rqs[0], &boot_task);

for (uint32_t i = 1; i < cpu_scheduler_count; i++) {
cpu_rqs[i].idle = idle_task_alloc(i);
Expand All @@ -1259,6 +1285,8 @@ void sched_init(void)
ap_boot_tasks[i].weight = SCHED_NICE_0_LOAD;
ilist_init(&ap_boot_tasks[i].sched_node);
ilist_init(&ap_boot_tasks[i].timer_node);

/* CPU i's GS is not reachable here; sched_ap_start() commits both sides. */
__atomic_store_n(&cpu_rqs[i].curr, &ap_boot_tasks[i], __ATOMIC_RELAXED);
}
}
Expand All @@ -1278,8 +1306,8 @@ __attribute__((noreturn)) void sched_ap_start(uint32_t cpu_id)
if (cpu_id == 0 || cpu_id >= cpu_scheduler_count) krn_halt();

while (!__atomic_load_n(&scheduler.started, __ATOMIC_ACQUIRE)) cpu_relax();
__atomic_store_n(&cpu_rqs[cpu_id].curr, &ap_boot_tasks[cpu_id], __ATOMIC_RELAXED);
percpu_gs_set_current(&ap_boot_tasks[cpu_id]);
local_rq_set_curr(&cpu_rqs[cpu_id], &ap_boot_tasks[cpu_id]);

sched_yield();
panic("sched: AP scheduler exited.");
}
Expand Down Expand Up @@ -1381,8 +1409,19 @@ int task_set_cpu(task_t *task, uint32_t cpu_id)
/* Switch to the next runnable task on the current CPU */
static void sched_switch(bool voluntary)
{
eevdf_rq_t *rq = local_rq();
uint64_t entry_rflags = spin_lock_irqsave(&rq->lock);
/*
* Disable interrupts before resolving the local runqueue. sched_yield()
* is entered with interrupts enabled, and a preemption plus migration
* between local_rq() and the lock would make this task operate on its old
* CPU's runqueue: it would treat that CPU's running task as prev and save
* its own registers into that task's context.
*/
uint64_t entry_rflags = get_rflags();
disable_intr();
eevdf_rq_t *rq = local_rq();

/* Interrupts are already off; entry_rflags owns the restore. */
(void)spin_lock_irqsave(&rq->lock);

__atomic_store_n(&rq->need_resched, 0, __ATOMIC_RELAXED);

Expand Down Expand Up @@ -1418,7 +1457,7 @@ static void sched_switch(bool voluntary)
*/
if (next == rq->idle && rq->nr_running == 0) {
/* Keep IRQs disabled while rq->curr is between scheduling states. */
spin_unlock_irqrestore(&rq->lock, entry_rflags & ~(1ULL << 9));
rq_unlock_irqs_off(rq, entry_rflags);
spin_lock(&scheduler.lock);
task_t *stolen = newidle_balance_locked(get_current_cpu_id());
(void)spin_lock_irqsave(&rq->lock);
Expand All @@ -1434,7 +1473,8 @@ static void sched_switch(bool voluntary)
next->time_slice = 0;
advance_min_vruntime(rq);
}
spin_unlock_irqrestore(&rq->lock, entry_rflags);
rq_unlock_irqs_off(rq, entry_rflags);
restore_intr(entry_rflags);
return;
}

Expand Down Expand Up @@ -1468,10 +1508,7 @@ static void sched_switch(bool voluntary)
if (!next->context.rsp) panic("sched: selected task %llu (%s) has no kernel stack (runqueue corruption?)", next->pid, next->name);

__atomic_store_n(&next->on_cpu, 1, __ATOMIC_RELEASE);
__atomic_store_n(&rq->curr, next, __ATOMIC_RELAXED);

/* Keep the GS-relative current pointer in lockstep with rq->curr. */
percpu_gs_set_current(next);
local_rq_set_curr(rq, next);
advance_min_vruntime(rq);
update_tss_stack(next);

Expand All @@ -1491,7 +1528,7 @@ static void sched_switch(bool voluntary)
* IF here would let a tick enter sched_switch() after rq->curr had changed but
* before the CPU had changed stacks.
*/
spin_unlock_irqrestore(&rq->lock, entry_rflags & ~(1ULL << 9));
rq_unlock_irqs_off(rq, entry_rflags);

/*
* arch_prctl keeps the software values authoritative, so avoid two
Expand All @@ -1508,11 +1545,7 @@ static void sched_switch(bool voluntary)
context_switch(&prev->context, &next->context, &prev->on_cpu);

/* Restore the IRQ state of the call site when this task is scheduled in. */
if (entry_rflags & (1ULL << 9)) {
enable_intr();
} else {
disable_intr();
}
restore_intr(entry_rflags);
}

/* Yield the current task to the scheduler */
Expand Down Expand Up @@ -2096,18 +2129,28 @@ void sched_tick(bool user_mode)
void sched_maybe_preempt(void)
{
if (!__atomic_load_n(&scheduler.started, __ATOMIC_ACQUIRE) || !cpu_rqs) return;
eevdf_rq_t *rq = local_rq();
if (!__atomic_load_n(&rq->need_resched, __ATOMIC_ACQUIRE)) return;

bool preempt = false;
spin_lock(&rq->lock);
if (__atomic_exchange_n(&rq->need_resched, 0, __ATOMIC_ACQ_REL)) {
task_t *curr = rq->curr;
if (curr && __atomic_load_n(&rq->nr_running, __ATOMIC_RELAXED) > 0) preempt = curr == rq->idle || (curr->state == TASK_RUNNING && pick_eevdf(rq) != curr);
/* Fast path: a racy hint, re-resolved below with interrupts off. */
uint32_t cpu_id = percpu_gs_cpu_id();
if (cpu_id >= cpu_scheduler_count || !__atomic_load_n(&cpu_rqs[cpu_id].need_resched, __ATOMIC_ACQUIRE)) return;

/* Resolve the runqueue with interrupts off so a migration cannot consume another CPU's request. */
uint64_t rflags = get_rflags();
disable_intr();

eevdf_rq_t *rq = local_rq();
bool preempt = false;
if (__atomic_load_n(&rq->need_resched, __ATOMIC_ACQUIRE)) {
spin_lock(&rq->lock);
if (__atomic_exchange_n(&rq->need_resched, 0, __ATOMIC_ACQ_REL)) {
task_t *curr = rq->curr;
if (curr && __atomic_load_n(&rq->nr_running, __ATOMIC_RELAXED) > 0) preempt = curr == rq->idle || (curr->state == TASK_RUNNING && pick_eevdf(rq) != curr);
}
spin_unlock(&rq->lock);
}
spin_unlock(&rq->lock);

if (preempt) sched_switch(false);
restore_intr(rflags);
}

/* sched_ticks - return the global tick count */
Expand Down
Loading