From 1e05d278b1f1c90269a80f1491b7844b7af008ea Mon Sep 17 00:00:00 2001 From: mylastresort Date: Sat, 22 Aug 2026 09:45:54 +0100 Subject: [PATCH 1/9] fix: autostart on config reload --- compose.yaml | 1 + internal/job/job.go | 8 +++++++- taskmaster.toml | 21 +-------------------- 3 files changed, 9 insertions(+), 21 deletions(-) diff --git a/compose.yaml b/compose.yaml index 0bed521..4c2d818 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,3 +9,4 @@ services: - "8080:8080" volumes: - /dev/log:/dev/log + - ./taskmaster.toml:/app/taskmaster.toml \ No newline at end of file diff --git a/internal/job/job.go b/internal/job/job.go index 04d454e..72ed175 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -380,7 +380,13 @@ func (j *Job) reread(prog *config.Program) bool { j.StdoutLogFile = prog.StdoutLogFile } - j.Autostart = prog.Autostart + if j.Autostart != prog.Autostart { + if !j.IsRunning() && prog.Autostart { + shouldRestart = true + } else if j.IsRunning() && !prog.Autostart { + shouldRestart = true + } + } j.ExitCodes = prog.ExitCodes j.StopWaitSecs = prog.StopWaitSecs j.StopSignal = utils.ParseSignal(prog.StopSignal) diff --git a/taskmaster.toml b/taskmaster.toml index 4a50806..be442a1 100644 --- a/taskmaster.toml +++ b/taskmaster.toml @@ -4,23 +4,4 @@ command = "ping localhost" stopsignal = "TERM" stdout_logfile = "one.log" -# autostart = false - -[program.two] -command = "ping 127.0.0.1" -stopsignal = "INT" -stdout_logfile = "two.log" -# autostart = false -startsecs = 5 -startretries = 10 - -[program.three] -# autostart = false -command = "ping 0.0.0.0" -stdout_logfile = "three.log" - -[program.four] -# autostart = false -command = "ping google.com" -stdout_logfile = "four.log" -numprocs = 3 +autostart = false \ No newline at end of file From 4b235f03d77a3d99eb11790da9fc08a2550fd4b9 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Mon, 24 Aug 2026 09:49:56 +0100 Subject: [PATCH 2/9] fix: reload lifecycle, per-process pgid tracking, and start/stop races --- .vscode/launch.json | 13 ++ Dockerfile | 2 +- cmd/client/main.go | 11 +- cmd/server/main.go | 7 + go.mod | 12 +- go.sum | 2 + internal/job/job.go | 322 +++---------------------------- internal/job/reload.go | 190 ++++++++++++++++++ internal/job/restart.go | 11 ++ internal/job/start.go | 197 +++++++++++++++++++ internal/job/states.go | 17 +- internal/job/stop.go | 110 +++++++++++ internal/logger/logger.go | 19 +- internal/manager/manager.go | 42 ++-- internal/manager/signals.go | 12 +- internal/parser/config/config.go | 38 +++- internal/server/server.go | 14 ++ internal/utils/setup.go | 2 +- setup.toml | 1 - taskmaster.toml | 8 +- 20 files changed, 693 insertions(+), 337 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 internal/job/reload.go create mode 100644 internal/job/restart.go create mode 100644 internal/job/start.go create mode 100644 internal/job/stop.go diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..b058a0d --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,13 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug server", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/cmd/server", + "cwd": "${workspaceFolder}" + } + ] +} diff --git a/Dockerfile b/Dockerfile index a5a9c77..afba8a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23.4 +FROM golang:1.25.14 WORKDIR /app diff --git a/cmd/client/main.go b/cmd/client/main.go index 3a08618..241e316 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -20,7 +20,16 @@ func main() { return } - client, err := client.NewClient(setup.Socket) + socket := setup.Socket + if len(os.Args) > 1 { + socket = os.Args[1] + } + if socket == "" { + fmt.Fprintln(os.Stderr, "usage: go run ./cmd/client (path printed by taskmasterd)") + return + } + + client, err := client.NewClient(socket) if err != nil { utils.Errorf(err.Error()) return diff --git a/cmd/server/main.go b/cmd/server/main.go index 6ac9c22..93f3aa6 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,6 +1,8 @@ package main import ( + "fmt" + "os" "sync" "github.com/Archer-01/taskmaster/internal/logger" @@ -17,6 +19,11 @@ func main() { logger.Critical(err) } + if setup.Socket == "" { + setup.Socket = fmt.Sprintf("/tmp/taskmasterd-%d.sock", os.Getpid()) + } + logger.Infof("Socket: %s", setup.Socket) + var wg sync.WaitGroup defer wg.Wait() diff --git a/go.mod b/go.mod index bb8819e..0f671f7 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,10 @@ module github.com/Archer-01/taskmaster go 1.23.4 -require github.com/BurntSushi/toml v1.4.0 // indirect -require ( - github.com/chzyer/readline v1.5.1 // indirect - golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect -) +toolchain go1.25.14 + +require github.com/BurntSushi/toml v1.4.0 + +require github.com/chzyer/readline v1.5.1 + +require golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect diff --git a/go.sum b/go.sum index 3bfcfb2..6a5f3e1 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,10 @@ github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 h1:y/woIyUBFbpQGKS0u1aHF/40WUDnek3fPOyD08H5Vng= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/job/job.go b/internal/job/job.go index 72ed175..5df26c6 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -3,17 +3,19 @@ package job import ( "fmt" "io" - "os" "os/exec" + "strconv" "sync" "syscall" - "time" - "github.com/Archer-01/taskmaster/internal/logger" "github.com/Archer-01/taskmaster/internal/parser/config" "github.com/Archer-01/taskmaster/internal/utils" ) +// serializes umask switching around process starts, since umask is +// process-wide and concurrent starts could inherit the wrong one +var startMu sync.Mutex + type Job struct { Name string Command string @@ -38,24 +40,24 @@ type Job struct { StdoutWriter *utils.DynamicWriter StderrWriter *utils.DynamicWriter NumProcs int - pgid int - startReady chan struct{} - startOnce sync.Once + _NumProcs int + pgid []int + startReady []chan struct{} + startOnce []sync.Once mustop sync.Mutex } -func NewJob(name string, prog *config.Program) *Job { - has_zero := false - exit_codes := prog.ExitCodes - for _, exit := range exit_codes { +func normalizeExitCodes(codes []int) []int { + for _, exit := range codes { if exit == 0 { - has_zero = true - break + return codes } } - if !has_zero { - exit_codes = append(exit_codes, 0) - } + return append(codes, 0) +} + +func NewJob(name string, prog *config.Program) *Job { + exit_codes := normalizeExitCodes(prog.ExitCodes) states := make([]string, prog.NumProcs) for i := range states { @@ -94,18 +96,15 @@ func NewJob(name string, prog *config.Program) *Job { StdoutWriter: &utils.DynamicWriter{}, StderrWriter: &utils.DynamicWriter{}, NumProcs: prog.NumProcs, + _NumProcs: prog.NumProcs, cmds: make([]*exec.Cmd, prog.NumProcs), - pgid: 0, - startReady: ch, + pgid: make([]int, prog.NumProcs), + startReady: make([]chan struct{}, prog.NumProcs), + startOnce: make([]sync.Once, prog.NumProcs), + mustop: sync.Mutex{}, } } -func (j *Job) closeStartReady() { - j.startOnce.Do(func() { - close(j.startReady) - }) -} - func (j *Job) DisplayName(procId int) string { if j.ProcessName != "" { return fmt.Sprintf(j.ProcessName, j.Name, procId) @@ -116,112 +115,10 @@ func (j *Job) DisplayName(procId int) string { return fmt.Sprintf("%s_%d", j.Name, procId) } -type WorkerFn = func(j *Job, wg *sync.WaitGroup, _done chan bool) error - -func (j *Job) Start(wg *sync.WaitGroup, _done chan bool) error { - defer func() { _done <- true }() - - j.mustop.Lock() - defer j.mustop.Unlock() - - startReady := make(chan struct{}) - j.startReady = startReady - j.startOnce = sync.Once{} - - for i := range j.NumProcs { - if j.Is(STOPPING, i) || j._running[i] { - continue - } - - j._running[i] = true - - if j.HasPgid() { - go j.startJobWorker(wg, i, j.pgid) - continue - } - - go j.startJobWorker(wg, i, 0) - <-startReady - if !j.Is(RUNNING, 0) { - return fmt.Errorf("process could not be running") - } - j.pgid = j.cmds[i].Process.Pid - } - - return nil -} - -func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { - wg.Add(1) - defer wg.Done() - - retries := 0 - for { - usePgid := pgid - if j.cmds[id] != nil && j.cmds[id].Process != nil { - usePgid = 0 - } - - cmd := exec.Command("sh", "-c", fmt.Sprintf("umask %v && %v", j.Umask, j.Command)) - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pgid: usePgid} - j.cmds[id] = cmd - - j.SetState(STARTING, id) - err := j.tryStart(id) - if err != nil { - logger.Error(err) - j.SetState(BACKOFF, id) - j.closeStartReady() - retries++ - if j.StartRetries == retries { - break - } - time.Sleep(1 * time.Second) - continue - } - - cur_ts := int(time.Now().Unix()) - j.SetState(RUNNING, id) - j.closeStartReady() - state, _ := j.cmds[id].Process.Wait() - j.cmds[id].ProcessState = state - - if j.Is(STOPPING, id) { - break - } else if int(time.Now().Unix())-cur_ts < j.StartSecs { - j.SetState(BACKOFF, id) - retries++ - if j.StartRetries == retries { - break - } - time.Sleep(1 * time.Second) - continue - } +type WorkerFn = func(j *Job, wg *sync.WaitGroup, _done chan bool, procId int, count int) error - j.SetState(EXITED, id) - retries = 0 - if j.Autorestart == AUTORESTART_FALSE { - break - } - if j.Autorestart == AUTORESTART_UNEXPECTED { - expected := false - for _, exit := range j.ExitCodes { - if exit == j.cmds[id].ProcessState.ExitCode() { - expected = true - break - } - } - if expected { - break - } - } - } - if j.Is(BACKOFF, id) { - j.SetState(FATAL, id) - } else if j.Is(STOPPING, id) { - j.SetState(STOPPED, id) - } - j._running[id] = false +func groupAlive(pgid int) bool { + return pgid > 0 && syscall.Kill(-pgid, 0) == nil } func (j *Job) setLog(file string, writer *utils.DynamicWriter, _default io.Writer) error { @@ -238,170 +135,9 @@ func (j *Job) setLog(file string, writer *utils.DynamicWriter, _default io.Write return nil } -func (j *Job) tryStart(procId int) error { - err := j.setLog(j.StdoutLogFile, j.StdoutWriter, os.Stdout) - if err != nil { - return err - } - - err = j.setLog(j.StderrLogFile, j.StderrWriter, os.Stderr) - if err != nil { - return err - } - - j.cmds[procId].Stdout = j.StdoutWriter - if j.RedirectStderr { - j.cmds[procId].Stderr = j.StdoutWriter - } else { - j.cmds[procId].Stderr = j.StderrWriter - } - - j.cmds[procId].Env = append(j.Environment, os.Environ()...) - j.cmds[procId].Dir = j.Dir - - err = j.cmds[procId].Start() - if err != nil { - return err - } - - return nil -} - -func (j *Job) Restart(wg *sync.WaitGroup, _done chan bool) error { - done := make(chan bool, 1) - defer close(done) - j.Stop(wg, done) - j.Start(wg, _done) - return nil -} - -func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool) error { - defer func() { _done <- true }() - <-j.startReady - j.mustop.Lock() - defer j.mustop.Unlock() - - if j.HasPgid() { - for i := range j.NumProcs { - j.SetState(STOPPING, i) - } - - err := syscall.Kill(-j.pgid, j.StopSignal) - if err != nil { - return err - } - - cur := time.Now().Unix() - for time.Now().Unix()-cur < int64(j.StopWaitSecs) && j.IsRunning() { - time.Sleep(100 * time.Millisecond) - } - - if j.HasPgid() && j.IsRunning() { - err = syscall.Kill(-j.pgid, syscall.SIGKILL) - if err != nil { - return err - } - } - - for j.IsRunning() { - time.Sleep(100 * time.Millisecond) - } - } - - j.SetPgid(0) - return nil -} - -func (j *Job) Reload(wg *sync.WaitGroup, _done chan bool, prog *config.Program) error { - wg.Add(1) - defer wg.Done() - - stdoutChanged := j.StdoutLogFile != prog.StdoutLogFile - stderrChanged := j.StderrLogFile != prog.StderrLogFile - shouldRestart := j.reread(prog) - if shouldRestart && j.IsRunning() { - go j.Restart(wg, _done) - return nil - } - - if stdoutChanged { - j.setLog(j.StdoutLogFile, j.StdoutWriter, os.Stdout) - } - - if stderrChanged { - j.setLog(j.StderrLogFile, j.StderrWriter, os.Stderr) - } - - _done <- true - return nil -} - -func (j *Job) reread(prog *config.Program) bool { - shouldRestart := false - - if prog.Command != j.Command { - j.Command = prog.Command - shouldRestart = true +func parseUmask(s string) int { + if v, err := strconv.ParseUint(s, 8, 32); err == nil { + return int(v) } - - if prog.Directory != j.Dir { - j.Dir = prog.Directory - shouldRestart = true - } - - { - table := make(map[string]int, len(j.Environment)) - for _, env := range j.Environment { - table[env] += 1 - } - for _, env := range prog.Environment { - table[env] += 1 - } - for _, c := range table { - if c != 2 { - shouldRestart = true - j.Environment = prog.Environment - break - } - } - - } - - if prog.Umask != j.Umask { - j.Umask = prog.Umask - shouldRestart = true - } - - if prog.StderrLogFile != j.StderrLogFile { - j.StderrLogFile = prog.StderrLogFile - } - - if prog.StdoutLogFile != j.StdoutLogFile { - j.StdoutLogFile = prog.StdoutLogFile - } - - if j.Autostart != prog.Autostart { - if !j.IsRunning() && prog.Autostart { - shouldRestart = true - } else if j.IsRunning() && !prog.Autostart { - shouldRestart = true - } - } - j.ExitCodes = prog.ExitCodes - j.StopWaitSecs = prog.StopWaitSecs - j.StopSignal = utils.ParseSignal(prog.StopSignal) - j.Autorestart = prog.Autorestart - j.StartSecs = prog.StartSecs - j.StartRetries = prog.StartRetries - j.Priority = prog.Priority - j.ProcessName = prog.ProcessName - - if prog.RedirectStderr != j.RedirectStderr { - j.RedirectStderr = prog.RedirectStderr - if j.IsRunning() { - shouldRestart = true - } - } - - return shouldRestart + return 0o022 } diff --git a/internal/job/reload.go b/internal/job/reload.go new file mode 100644 index 0000000..55c397f --- /dev/null +++ b/internal/job/reload.go @@ -0,0 +1,190 @@ +package job + +import ( + "os" + "os/exec" + "sync" + + "github.com/Archer-01/taskmaster/internal/logger" + "github.com/Archer-01/taskmaster/internal/parser/config" + "github.com/Archer-01/taskmaster/internal/utils" +) + +func (j *Job) reread(prog *config.Program) (shouldRestart bool, shouldStop bool, shouldStart bool, numprocsChanged int) { + shouldRestart = false + shouldStop = false + shouldStart = false + numprocsChanged = 0 + + if prog.Command != j.Command { + j.Command = prog.Command + shouldRestart = true + } + + if prog.Directory != j.Dir { + j.Dir = prog.Directory + shouldRestart = true + } + + { + table := make(map[string]int, len(j.Environment)) + for _, env := range j.Environment { + table[env] += 1 + } + for _, env := range prog.Environment { + table[env] += 1 + } + for _, c := range table { + if c != 2 { + shouldRestart = true + j.Environment = prog.Environment + break + } + } + + } + + if prog.Umask != j.Umask { + j.Umask = prog.Umask + shouldRestart = true + } + + if prog.StderrLogFile != j.StderrLogFile { + j.StderrLogFile = prog.StderrLogFile + shouldRestart = true + } + + if prog.StdoutLogFile != j.StdoutLogFile { + j.StdoutLogFile = prog.StdoutLogFile + shouldRestart = true + } + + if j.Autostart != prog.Autostart { + j.Autostart = prog.Autostart + if !j.IsRunning() && prog.Autostart { + shouldStart = true + } else if j.IsRunning() && !prog.Autostart { + shouldStop = true + } + } + j.ExitCodes = normalizeExitCodes(prog.ExitCodes) + j.StopWaitSecs = prog.StopWaitSecs + j.StopSignal = utils.ParseSignal(prog.StopSignal) + j.Autorestart = prog.Autorestart + j.StartSecs = prog.StartSecs + j.StartRetries = prog.StartRetries + j.Priority = prog.Priority + j.ProcessName = prog.ProcessName + + if prog.RedirectStderr != j.RedirectStderr { + j.RedirectStderr = prog.RedirectStderr + if j.IsRunning() { + shouldRestart = true + } + } + + if prog.NumProcs != j.NumProcs { + numprocsChanged = prog.NumProcs - j.NumProcs + j._NumProcs = prog.NumProcs + } + + return shouldRestart, shouldStop, shouldStart, numprocsChanged +} + +func (j *Job) Resize(newSize int) { + l := len(j._running) + if l > newSize { + j._running = j._running[:newSize] + j.State = j.State[:newSize] + j.cmds = j.cmds[:newSize] + j.pgid = j.pgid[:newSize] + j.startReady = j.startReady[:newSize] + j.startOnce = j.startOnce[:newSize] + } else if l < newSize { + cmds := make([]*exec.Cmd, newSize) + pgid := make([]int, newSize) + startReady := make([]chan struct{}, newSize) + startOnce := make([]sync.Once, newSize) + states := make([]string, newSize) + + for i := range states { + states[i] = STOPPED + } + + running := make([]bool, newSize) + + for i := range running { + running[i] = false + } + + copy(cmds, j.cmds) + copy(pgid, j.pgid) + copy(startReady, j.startReady) + copy(startOnce, j.startOnce) + copy(states, j.State) + copy(running, j._running) + + j.cmds = cmds + j.pgid = pgid + j.startReady = startReady + j.startOnce = startOnce + j.State = states + j._running = running + } +} +func (j *Job) Reload(wg *sync.WaitGroup, _done chan bool, prog *config.Program) error { + wg.Add(1) + defer wg.Done() + + stdoutChanged := j.StdoutLogFile != prog.StdoutLogFile + stderrChanged := j.StderrLogFile != prog.StderrLogFile + shouldRestart, shouldStop, shouldStart, numprocsChanged := j.reread(prog) + skipedDone := true + if j.IsRunning() { + if shouldStop { + skipedDone = false + go j.Stop(wg, _done, -1, 1) + } else if shouldRestart { + skipedDone = false + go j.Restart(wg, _done, -1, 1) + } + num := 0 + var ch chan bool + if numprocsChanged > 0 { + ch = make(chan bool, 1) + num = numprocsChanged + j.NumProcs = j._NumProcs + j.Resize(j._NumProcs) + go j.Start(wg, ch, j.NumProcs-numprocsChanged, numprocsChanged) + } else if numprocsChanged < 0 { + ch = make(chan bool, 1) + num = -numprocsChanged + go j.Stop(wg, ch, j.NumProcs+numprocsChanged, num) + } + if num != 0 { + <-ch + } + if numprocsChanged < 0 { + logger.Debugf("Reload: resizing job to %d", j._NumProcs) + j.Resize(j._NumProcs) + j.NumProcs = j._NumProcs + logger.Debug("Reload: resize done") + } + } else if shouldStart { + skipedDone = false + go j.Start(wg, _done, -1, 1) + } + + if stdoutChanged { + j.setLog(j.StdoutLogFile, j.StdoutWriter, os.Stdout) + } + + if stderrChanged { + j.setLog(j.StderrLogFile, j.StderrWriter, os.Stderr) + } + + if skipedDone { + _done <- true + } + return nil +} diff --git a/internal/job/restart.go b/internal/job/restart.go new file mode 100644 index 0000000..29f1873 --- /dev/null +++ b/internal/job/restart.go @@ -0,0 +1,11 @@ +package job + +import "sync" + +func (j *Job) Restart(wg *sync.WaitGroup, _done chan bool, procId int, count int) error { + done := make(chan bool, 1) + defer close(done) + j.Stop(wg, done, procId, count) + j.Start(wg, _done, procId, count) + return nil +} diff --git a/internal/job/start.go b/internal/job/start.go new file mode 100644 index 0000000..2976bd0 --- /dev/null +++ b/internal/job/start.go @@ -0,0 +1,197 @@ +package job + +import ( + "os" + "os/exec" + "sync" + "syscall" + "time" + + "github.com/Archer-01/taskmaster/internal/logger" + "github.com/Archer-01/taskmaster/internal/parser/config" +) + +func (j *Job) closeStartReady(procId int) { + j.startOnce[procId].Do(func() { + close(j.startReady[procId]) + }) +} + +func (j *Job) Start(wg *sync.WaitGroup, _done chan bool, procId int, count int) error { + logger.Debugf("Starting job %s, procId = %d, NumProcs = %d, j._NumProcs = %d", j.Name, procId, j.NumProcs, j._NumProcs) + defer func() { _done <- true }() + + j.mustop.Lock() + defer j.mustop.Unlock() + + st := func(i int) { + j.startReady[i] = make(chan struct{}) + j.startOnce[i] = sync.Once{} + if j.Is(STOPPING, i) || j._running[i] { + return + } + + j._running[i] = true + + if j.HasPgid(i) { + pgid := j.pgid[i] + if !groupAlive(pgid) { + pgid = 0 + j.pgid[i] = 0 + } + go j.startJobWorker(wg, i, pgid) + return + } + + go j.startJobWorker(wg, i, 0) + } + + logger.Debugf("Starting job %s, procId = %d, NumProcs = %d, j._NumProcs = %d", j.Name, procId, j.NumProcs, j._NumProcs) + if procId >= 0 && procId < j.NumProcs { + for i := procId; i < procId+count && i < j.NumProcs; i++ { + logger.Infof("Starting process %s", j.DisplayName(i)) + st(i) + } + } else { + logger.Infof("Starting all processes of job %s", j.Name) + for i := range j.NumProcs { + st(i) + } + } + + if procId >= 0 && procId < j.NumProcs { + <-j.startReady[procId] + } else { + for i := range j.NumProcs { + <-j.startReady[i] + } + } + return nil +} + +func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { + wg.Add(1) + defer wg.Done() + defer j.closeStartReady(id) + + retries := 0 + for { + if j.Is(STOPPING, id) { + break + } + + usePgid := pgid + if usePgid != 0 && !groupAlive(usePgid) { + usePgid = 0 + } + if j.cmds[id] != nil && j.cmds[id].Process != nil { + usePgid = 0 + } + + argv := config.ParseCommand(j.Command) + if len(argv) == 0 { + argv = []string{j.Command} + } + cmd := exec.Command(argv[0], argv[1:]...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pgid: usePgid} + j.cmds[id] = cmd + + j.SetState(STARTING, id) + err := j.tryStart(id) + if err != nil { + logger.Error(err) + j.SetState(BACKOFF, id) + j.closeStartReady(id) + retries++ + if j.StartRetries == retries { + break + } + time.Sleep(1 * time.Second) + continue + } + + cur_ts := int(time.Now().Unix()) + j.SetState(RUNNING, id) + if usePgid == 0 && j.cmds[id].Process != nil { + j.pgid[id], _ = syscall.Getpgid(j.cmds[id].Process.Pid) + } + j.closeStartReady(id) + err = j.cmds[id].Wait() + if err != nil { + logger.Error(err) + } + + if j.Is(STOPPING, id) { + break + } else if int(time.Now().Unix())-cur_ts < j.StartSecs { + j.SetState(BACKOFF, id) + retries++ + if j.StartRetries == retries { + break + } + time.Sleep(1 * time.Second) + continue + } + + j.SetState(EXITED, id) + retries = 0 + if j.Autorestart == AUTORESTART_FALSE { + break + } + if j.Autorestart == AUTORESTART_UNEXPECTED { + expected := false + exitCode := -1 + if j.cmds[id].ProcessState != nil { + exitCode = j.cmds[id].ProcessState.ExitCode() + } + for _, exit := range j.ExitCodes { + if exit == exitCode { + expected = true + break + } + } + if expected { + break + } + } + } + if j.Is(BACKOFF, id) { + j.SetState(FATAL, id) + } else if j.Is(STOPPING, id) { + j.SetState(STOPPED, id) + } + j._running[id] = false +} + +func (j *Job) tryStart(procId int) error { + err := j.setLog(j.StdoutLogFile, j.StdoutWriter, os.Stdout) + if err != nil { + return err + } + + err = j.setLog(j.StderrLogFile, j.StderrWriter, os.Stderr) + if err != nil { + return err + } + + j.cmds[procId].Stdout = j.StdoutWriter + if j.RedirectStderr { + j.cmds[procId].Stderr = j.StdoutWriter + } else { + j.cmds[procId].Stderr = j.StderrWriter + } + + j.cmds[procId].Env = append(j.Environment, os.Environ()...) + j.cmds[procId].Dir = j.Dir + + startMu.Lock() + old := syscall.Umask(parseUmask(j.Umask)) + err = j.cmds[procId].Start() + syscall.Umask(old) + startMu.Unlock() + if err != nil { + return err + } + + return nil +} diff --git a/internal/job/states.go b/internal/job/states.go index ae08059..7a536d9 100644 --- a/internal/job/states.go +++ b/internal/job/states.go @@ -28,7 +28,7 @@ func (j *Job) SetState(state string, procId int) error { case STOPPED: j.State[procId] = STOPPED if true { - j.pgid = 0 + j.pgid[procId] = 0 } default: return fmt.Errorf("invalid state: %s", state) @@ -40,12 +40,12 @@ func (j *Job) Is(state string, procId int) bool { return j.State[procId] == state } -func (j *Job) HasPgid() bool { - return j.pgid != 0 +func (j *Job) HasPgid(procId int) bool { + return j.pgid[procId] != 0 } -func (j *Job) SetPgid(num int) { - j.pgid = num +func (j *Job) SetPgid(procId int, num int) { + j.pgid[procId] = num } func (j *Job) IsRunning() bool { @@ -56,3 +56,10 @@ func (j *Job) IsRunning() bool { } return false } + +func (j *Job) procAlive(id int) bool { + if !j._running[id] || !j.HasPgid(id) { + return false + } + return groupAlive(j.pgid[id]) +} diff --git a/internal/job/stop.go b/internal/job/stop.go new file mode 100644 index 0000000..d71c4d8 --- /dev/null +++ b/internal/job/stop.go @@ -0,0 +1,110 @@ +package job + +import ( + "sync" + "syscall" + "time" + + "github.com/Archer-01/taskmaster/internal/logger" +) + +// startProcId = -1 means stop all processes +// startProcId >= 0 means stop the process with the given procId +// countProcId is the number of processes to stop, starting from startProcId +func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countProcId int) error { + defer func() { _done <- true }() + + if countProcId > j.NumProcs-1 { + countProcId = j.NumProcs - 1 + } else if countProcId < 0 { + countProcId = 1 + } + + logger.Debugf("Stopping job %s, procId = %d, NumProcs = %d, j._NumProcs = %d", j.Name, startProcId, j.NumProcs, j._NumProcs) + j.mustop.Lock() + defer j.mustop.Unlock() + + logger.Debugf( + "procId = %d, NumProcs = %d, j._NumProcs = %d", startProcId, j.NumProcs, j._NumProcs, + ) + if startProcId >= 0 && startProcId < j.NumProcs { + for i := startProcId; i < startProcId+countProcId && i < j.NumProcs; i++ { + logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) + <-j.startReady[i] + logger.Debugf("Stop(): Process %s is ready to stop", j.DisplayName(i)) + } + } else { + logger.Debugf("Stop(): Stopping all processes of job %s", j.Name) + for i := range j.NumProcs { + <-j.startReady[i] + } + } + + _wg := sync.WaitGroup{} + logger.Debugf("Stop(): All processes of job %s are ready to stop", j.Name) + + st := func(i int) { + defer _wg.Done() + logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) + if j.procAlive(i) { + logger.Infof("Stop(): Sending stop signal to process %s", j.DisplayName(i)) + j.SetState(STOPPING, i) + + err := syscall.Kill(-j.pgid[i], j.StopSignal) + logger.Debugf("Stop(): Sent stop signal to process %s", j.DisplayName(i)) + if err != nil && err != syscall.ESRCH { + logger.Error(err) + } + + cur := time.Now().Unix() + for time.Now().Unix()-cur < int64(j.StopWaitSecs) && j.procAlive(i) { + time.Sleep(100 * time.Millisecond) + } + + logger.Debugf("Stop(): Stop wait time elapsed for process %s", j.DisplayName(i)) + + if j.procAlive(i) { + logger.Warnf("Stop(): Sending SIGKILL to process %s", j.DisplayName(i)) + err = syscall.Kill(-j.pgid[i], syscall.SIGKILL) + logger.Debugf("Stop(): Sent SIGKILL to process %s", j.DisplayName(i)) + if err != nil && err != syscall.ESRCH { + logger.Error(err) + } + } + } + logger.Infof("Stop(): Process %s stopped", j.DisplayName(i)) + j.SetPgid(i, 0) + j._running[i] = false + j.SetState(STOPPED, i) + } + + if startProcId >= 0 && startProcId < j.NumProcs { + for i := startProcId; i < startProcId+countProcId && i < j.NumProcs; i++ { + logger.Infof("Stopping process %s", j.DisplayName(i)) + _wg.Add(1) + go st(i) + } + } else { + for i := range j.NumProcs { + logger.Infof("Stopping process %s", j.DisplayName(i)) + _wg.Add(1) + go st(i) + } + } + _wg.Wait() + + if startProcId >= 0 && startProcId < j.NumProcs { + for i := startProcId; i < startProcId+countProcId && i < j.NumProcs; i++ { + logger.Debugf("Stop(): Waiting for process %s to exit", j.DisplayName(i)) + j.cmds[i].Wait() + logger.Debugf("Stop(): Process %s exited", j.DisplayName(i)) + } + } else { + logger.Debugf("Stop(): Waiting for all processes of job %s to exit", j.Name) + for i := range j.NumProcs { + logger.Debugf("Stop(): Waiting for process %s to exit", j.DisplayName(i)) + j.cmds[i].Wait() + } + } + return nil +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 7025b28..0545219 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -11,7 +11,8 @@ import ( type LogLevel uint8 const ( - InfoLevel LogLevel = iota + DebugLevel LogLevel = iota + InfoLevel WarnLevel ErrorLevel CriticalLevel @@ -19,6 +20,8 @@ const ( func (lvl LogLevel) String() string { switch lvl { + case DebugLevel: + return "DBUG" case InfoLevel: return "INFO" case WarnLevel: @@ -62,6 +65,20 @@ func SetLevel(level LogLevel) { logger.level = level } +func Debug(a any) { + logger.log(DebugLevel, a) + + message := fmt.Sprintf("%v", a) + logger.syslogger.Debug(message) +} + +func Debugf(format string, a ...any) { + message := fmt.Sprintf(format, a...) + logger.log(DebugLevel, message) + + logger.syslogger.Debug(message) +} + func Info(a any) { logger.log(InfoLevel, a) diff --git a/internal/manager/manager.go b/internal/manager/manager.go index c085f20..2723ced 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -89,7 +89,7 @@ func (m *JobManager) reload() error { os.Exit(1) } - fmt.Println("[NOTICE] De-escalation successful") + logger.Info("[NOTICE] De-escalation successful") } stop := make([]chan bool, 0) @@ -98,7 +98,7 @@ func (m *JobManager) reload() error { if !fd { d := make(chan bool, 1) stop = append(stop, d) - go j.Stop(m.wg, d) + go j.Stop(m.wg, d, -1, 1) delete(m.Jobs, name) } } @@ -132,7 +132,7 @@ func (m *JobManager) reload() error { start = append(start, d) j := job.NewJob(nj.name, nj.prog) m.Jobs[nj.name] = j - go j.Start(m.wg, d) + go j.Start(m.wg, d, -1, 1) } for _, _done := range stop { @@ -176,7 +176,7 @@ func (m *JobManager) start() { done = make(chan bool, 1) defer close(done) logger.Infof("[STARTING] Program(name=%s)", j.Name) - j.Start(m.wg, done) + j.Start(m.wg, done, -1, 1) <-done } } @@ -194,23 +194,23 @@ func (m *JobManager) Run() { action.Done <- true return - case RELOAD: - logger.Warn("Reloading...") - if err := m.reload(); err != nil { - action.Data <- err.Error() - action.Done <- false - } else { - action.Done <- true - } + case RELOAD: + logger.Warn("Reloading...") + if err := m.reload(); err != nil { + action.Data <- err.Error() + action.Done <- false + } else { + action.Done <- true + } case START: - m.setJobs("STARTING", (*job.Job).Start, action) + m.setJobs("STARTING", (*job.Job).Start, action, -1, 1) case STOP: - m.setJobs("STOPPING", (*job.Job).Stop, action) + m.setJobs("STOPPING", (*job.Job).Stop, action, -1, 1) case RESTART: - m.setJobs("RESTARTING", (*job.Job).Restart, action) + m.setJobs("RESTARTING", (*job.Job).Restart, action, -1, 1) case STATUS: m.getStatus(action) @@ -222,9 +222,9 @@ func (m *JobManager) Run() { } } -func (m *JobManager) runWorkerJob(j *job.Job, worker job.WorkerFn, done chan bool, state string) { +func (m *JobManager) runWorkerJob(j *job.Job, worker job.WorkerFn, done chan bool, state string, procId int) { logger.Infof("[%s] Program(name=%s)", state, j.Name) - go worker(j, m.wg, done) + go worker(j, m.wg, done, procId, 1) } func (m *JobManager) runWorkerJobs(jobs []*job.Job, worker job.WorkerFn, action Action, state string) { @@ -232,7 +232,7 @@ func (m *JobManager) runWorkerJobs(jobs []*job.Job, worker job.WorkerFn, action for _, j := range jobs { _done := make(chan bool, 1) jobs_done = append(jobs_done, _done) - m.runWorkerJob(j, worker, _done, state) + m.runWorkerJob(j, worker, _done, state, -1) } for _, _done := range jobs_done { defer close(_done) @@ -241,7 +241,7 @@ func (m *JobManager) runWorkerJobs(jobs []*job.Job, worker job.WorkerFn, action action.Done <- true } -func (m *JobManager) setJobs(state string, worker job.WorkerFn, action Action) { +func (m *JobManager) setJobs(state string, worker job.WorkerFn, action Action, procId int, count int) { if len(action.Args) != 1 { action.Data <- "command accepts 1 argument only" action.Done <- false @@ -255,7 +255,7 @@ func (m *JobManager) setJobs(state string, worker job.WorkerFn, action Action) { action.Done <- false return } - m.runWorkerJob(j, worker, action.Done, state) + m.runWorkerJob(j, worker, action.Done, state, procId) } else { var sorted []*job.Job if state == "STOPPING" { @@ -324,7 +324,7 @@ func (m *JobManager) stop() { done = make(chan bool, 1) defer close(done) logger.Infof("[STOPPING] Program(name=%s)", j.Name) - j.Stop(m.wg, done) + j.Stop(m.wg, done, -1, 1) <-done } } diff --git a/internal/manager/signals.go b/internal/manager/signals.go index f04bc6b..d0f2a4a 100644 --- a/internal/manager/signals.go +++ b/internal/manager/signals.go @@ -34,11 +34,19 @@ func (m *JobManager) WaitForSignals(wg *sync.WaitGroup) { switch sig { case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT: - m.actions <- Action{Type: QUIT} + m.actions <- Action{ + Type: QUIT, + Done: make(chan bool, 1), + Data: make(chan string, 1), + } return case syscall.SIGHUP: - m.actions <- Action{Type: RELOAD} + m.actions <- Action{ + Type: RELOAD, + Done: make(chan bool, 1), + Data: make(chan string, 1), + } } } diff --git a/internal/parser/config/config.go b/internal/parser/config/config.go index 9750380..ce89033 100644 --- a/internal/parser/config/config.go +++ b/internal/parser/config/config.go @@ -32,9 +32,41 @@ type Config struct { } func ParseCommand(cmd string) []string { - return strings.FieldsFunc(cmd, func(r rune) bool { - return strings.ContainsRune(" \t\n\v\f\r", r) - }) + var args []string + var cur strings.Builder + inWord := false + quote := rune(0) + + flush := func() { + if inWord { + args = append(args, cur.String()) + cur.Reset() + inWord = false + } + } + + for _, r := range cmd { + switch { + case quote != 0: + if r == quote { + quote = 0 + } else { + cur.WriteRune(r) + } + inWord = true + case r == '\'' || r == '"': + quote = r + inWord = true + case strings.ContainsRune(" \t\n\v\f\r", r): + flush() + default: + cur.WriteRune(r) + inWord = true + } + } + flush() + + return args } func ParseConfig(file string) (Config, error) { diff --git a/internal/server/server.go b/internal/server/server.go index 742c7cf..0ffb6ed 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,6 +1,7 @@ package server import ( + "fmt" "net" "os" "sync" @@ -52,6 +53,19 @@ func (s *Server) Stop() { } func (s *Server) Init() error { + if _, err := os.Stat(s.addr); err == nil { + // Socket file already exists: only remove it if nothing is + // actually listening on it (stale leftover from a crash). + if con, derr := net.Dial("unix", s.addr); derr == nil { + con.Close() + return fmt.Errorf("another instance is already listening on %s", s.addr) + } + logger.Info("Removing stale socket " + s.addr) + if err := os.Remove(s.addr); err != nil { + return err + } + } + sock, err := net.Listen("unix", s.addr) if err != nil { return err diff --git a/internal/utils/setup.go b/internal/utils/setup.go index e2eef5d..592d9cf 100644 --- a/internal/utils/setup.go +++ b/internal/utils/setup.go @@ -11,7 +11,7 @@ import ( type Setup struct { Prompt string `toml:"prompt"` - Socket string `toml:"socket" validate:"default=/tmp/taskmaster.sock"` + Socket string `toml:"socket"` Config string `toml:"config" validate:"default=taskmaster.toml"` } diff --git a/setup.toml b/setup.toml index 68ac26d..39a7136 100644 --- a/setup.toml +++ b/setup.toml @@ -1,2 +1 @@ -socket = "/tmp/taskmaster.sock" prompt = "taskmaster> " diff --git a/taskmaster.toml b/taskmaster.toml index be442a1..bbb390e 100644 --- a/taskmaster.toml +++ b/taskmaster.toml @@ -1,7 +1,9 @@ # user = "taskmaster" [program.one] -command = "ping localhost" -stopsignal = "TERM" +command = "sleep 1000" +stopsignal = "INT" stdout_logfile = "one.log" -autostart = false \ No newline at end of file +autostart = true +startsecs = 120 +numprocs = 12 \ No newline at end of file From 672e18129f723e1d2dd32b3e52d1bfb489522081 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Mon, 24 Aug 2026 11:38:41 +0100 Subject: [PATCH 3/9] fix: stop/reload races with process mutex and LOG_LEVEL config --- compose.yaml | 2 ++ internal/job/job.go | 1 + internal/job/reload.go | 29 +++++++++++++---------------- internal/job/start.go | 4 ++-- internal/job/states.go | 5 +---- internal/job/stop.go | 20 +++++++++++++------- internal/logger/logger.go | 25 ++++++++++++++++++++++++- taskmaster.toml | 2 +- 8 files changed, 57 insertions(+), 31 deletions(-) diff --git a/compose.yaml b/compose.yaml index 4c2d818..2faff6e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -7,6 +7,8 @@ services: dockerfile: Dockerfile ports: - "8080:8080" + environment: + LOG_LEVEL: DEBUG volumes: - /dev/log:/dev/log - ./taskmaster.toml:/app/taskmaster.toml \ No newline at end of file diff --git a/internal/job/job.go b/internal/job/job.go index 5df26c6..8ce3f9f 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -45,6 +45,7 @@ type Job struct { startReady []chan struct{} startOnce []sync.Once mustop sync.Mutex + muproc sync.Mutex } func normalizeExitCodes(codes []int) []int { diff --git a/internal/job/reload.go b/internal/job/reload.go index 55c397f..549dc8f 100644 --- a/internal/job/reload.go +++ b/internal/job/reload.go @@ -148,27 +148,24 @@ func (j *Job) Reload(wg *sync.WaitGroup, _done chan bool, prog *config.Program) skipedDone = false go j.Restart(wg, _done, -1, 1) } - num := 0 - var ch chan bool if numprocsChanged > 0 { - ch = make(chan bool, 1) - num = numprocsChanged j.NumProcs = j._NumProcs j.Resize(j._NumProcs) - go j.Start(wg, ch, j.NumProcs-numprocsChanged, numprocsChanged) + go j.Start(wg, _done, j.NumProcs-numprocsChanged, numprocsChanged) + skipedDone = false } else if numprocsChanged < 0 { - ch = make(chan bool, 1) - num = -numprocsChanged + num := -numprocsChanged + ch := make(chan bool, 1) go j.Stop(wg, ch, j.NumProcs+numprocsChanged, num) - } - if num != 0 { - <-ch - } - if numprocsChanged < 0 { - logger.Debugf("Reload: resizing job to %d", j._NumProcs) - j.Resize(j._NumProcs) - j.NumProcs = j._NumProcs - logger.Debug("Reload: resize done") + go func() { + <-ch + logger.Debugf("Reload: resizing job to %d", j._NumProcs) + j.Resize(j._NumProcs) + j.NumProcs = j._NumProcs + logger.Debug("Reload: resize done") + _done <- true + }() + skipedDone = false } } else if shouldStart { skipedDone = false diff --git a/internal/job/start.go b/internal/job/start.go index 2976bd0..3f757bd 100644 --- a/internal/job/start.go +++ b/internal/job/start.go @@ -99,7 +99,7 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { j.SetState(STARTING, id) err := j.tryStart(id) if err != nil { - logger.Error(err) + logger.Debug(err) j.SetState(BACKOFF, id) j.closeStartReady(id) retries++ @@ -118,7 +118,7 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { j.closeStartReady(id) err = j.cmds[id].Wait() if err != nil { - logger.Error(err) + logger.Debug(err) } if j.Is(STOPPING, id) { diff --git a/internal/job/states.go b/internal/job/states.go index 7a536d9..8865393 100644 --- a/internal/job/states.go +++ b/internal/job/states.go @@ -58,8 +58,5 @@ func (j *Job) IsRunning() bool { } func (j *Job) procAlive(id int) bool { - if !j._running[id] || !j.HasPgid(id) { - return false - } - return groupAlive(j.pgid[id]) + return j._running[id] && j.HasPgid(id) && groupAlive(j.pgid[id]) } diff --git a/internal/job/stop.go b/internal/job/stop.go index d71c4d8..08a5d44 100644 --- a/internal/job/stop.go +++ b/internal/job/stop.go @@ -46,14 +46,18 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr st := func(i int) { defer _wg.Done() logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) - if j.procAlive(i) { - logger.Infof("Stop(): Sending stop signal to process %s", j.DisplayName(i)) + j.muproc.Lock() + if !j.procAlive(i) { + j.muproc.Unlock() + } else { + logger.Debugf("Stop(): Sending stop signal to process %s", j.DisplayName(i)) j.SetState(STOPPING, i) + j.muproc.Unlock() err := syscall.Kill(-j.pgid[i], j.StopSignal) logger.Debugf("Stop(): Sent stop signal to process %s", j.DisplayName(i)) if err != nil && err != syscall.ESRCH { - logger.Error(err) + logger.Debug(err) } cur := time.Now().Unix() @@ -68,25 +72,27 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr err = syscall.Kill(-j.pgid[i], syscall.SIGKILL) logger.Debugf("Stop(): Sent SIGKILL to process %s", j.DisplayName(i)) if err != nil && err != syscall.ESRCH { - logger.Error(err) + logger.Debug(err) } } } - logger.Infof("Stop(): Process %s stopped", j.DisplayName(i)) + j.muproc.Lock() + logger.Debugf("Stop(): Process %s stopped", j.DisplayName(i)) j.SetPgid(i, 0) j._running[i] = false j.SetState(STOPPED, i) + j.muproc.Unlock() } if startProcId >= 0 && startProcId < j.NumProcs { for i := startProcId; i < startProcId+countProcId && i < j.NumProcs; i++ { - logger.Infof("Stopping process %s", j.DisplayName(i)) + logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) _wg.Add(1) go st(i) } } else { for i := range j.NumProcs { - logger.Infof("Stopping process %s", j.DisplayName(i)) + logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) _wg.Add(1) go st(i) } diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 0545219..392e3d7 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -4,6 +4,7 @@ import ( "fmt" "log/syslog" "os" + "strings" "sync" "time" ) @@ -55,12 +56,34 @@ func Init() { os.Exit(1) } + level := InfoLevel + if lvl, ok := ParseLevel(os.Getenv("LOG_LEVEL")); ok { + level = lvl + } + logger = Logger{ - level: InfoLevel, + level: level, syslogger: syslogger, } } +func ParseLevel(name string) (LogLevel, bool) { + switch strings.ToUpper(strings.TrimSpace(name)) { + case "DEBUG": + return DebugLevel, true + case "INFO": + return InfoLevel, true + case "WARN", "WARNING": + return WarnLevel, true + case "ERROR": + return ErrorLevel, true + case "CRITICAL": + return CriticalLevel, true + default: + return InfoLevel, false + } +} + func SetLevel(level LogLevel) { logger.level = level } diff --git a/taskmaster.toml b/taskmaster.toml index bbb390e..d7dc9f9 100644 --- a/taskmaster.toml +++ b/taskmaster.toml @@ -6,4 +6,4 @@ stopsignal = "INT" stdout_logfile = "one.log" autostart = true startsecs = 120 -numprocs = 12 \ No newline at end of file +numprocs = 4 \ No newline at end of file From 2bee623c6b3ad8191036b16393e30f8232a0953d Mon Sep 17 00:00:00 2001 From: mylastresort Date: Mon, 24 Aug 2026 13:27:59 +0100 Subject: [PATCH 4/9] fix: track process uptime to honor startsecs before marking RUNNING --- internal/job/job.go | 17 ++++++++++- internal/job/reload.go | 68 ++++++++++++++++++++++++++---------------- internal/job/start.go | 39 ++++++++++++++++++++---- internal/job/stop.go | 10 +++---- taskmaster.toml | 35 ++++++++++++++++++++-- 5 files changed, 130 insertions(+), 39 deletions(-) diff --git a/internal/job/job.go b/internal/job/job.go index 8ce3f9f..64aa082 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -7,6 +7,7 @@ import ( "strconv" "sync" "syscall" + "time" "github.com/Archer-01/taskmaster/internal/parser/config" "github.com/Archer-01/taskmaster/internal/utils" @@ -16,6 +17,18 @@ import ( // process-wide and concurrent starts could inherit the wrong one var startMu sync.Mutex +func (p *Job) StartCmd(procId int) error { + if err := p.cmds[procId].Start(); err != nil { + return err + } + p.startTime[procId] = time.Now() + return nil +} + +func (p *Job) Uptime(procId int) time.Duration { + return time.Since(p.startTime[procId]) +} + type Job struct { Name string Command string @@ -23,6 +36,7 @@ type Job struct { Environment []string Dir string Autostart bool + startTime []time.Time StdoutLogFile string StderrLogFile string Umask string @@ -45,7 +59,7 @@ type Job struct { startReady []chan struct{} startOnce []sync.Once mustop sync.Mutex - muproc sync.Mutex + // muproc sync.Mutex } func normalizeExitCodes(codes []int) []int { @@ -102,6 +116,7 @@ func NewJob(name string, prog *config.Program) *Job { pgid: make([]int, prog.NumProcs), startReady: make([]chan struct{}, prog.NumProcs), startOnce: make([]sync.Once, prog.NumProcs), + startTime: make([]time.Time, prog.NumProcs), mustop: sync.Mutex{}, } } diff --git a/internal/job/reload.go b/internal/job/reload.go index 549dc8f..e483265 100644 --- a/internal/job/reload.go +++ b/internal/job/reload.go @@ -4,26 +4,36 @@ import ( "os" "os/exec" "sync" + "time" "github.com/Archer-01/taskmaster/internal/logger" "github.com/Archer-01/taskmaster/internal/parser/config" "github.com/Archer-01/taskmaster/internal/utils" ) -func (j *Job) reread(prog *config.Program) (shouldRestart bool, shouldStop bool, shouldStart bool, numprocsChanged int) { - shouldRestart = false - shouldStop = false - shouldStart = false - numprocsChanged = 0 +type ChangedState struct { + shouldRestart bool + shouldStop bool + shouldStart bool + numprocsChanged int +} + +func (j *Job) reread(prog *config.Program) ChangedState { + state := ChangedState{ + shouldRestart: false, + shouldStop: false, + shouldStart: false, + numprocsChanged: 0, + } if prog.Command != j.Command { j.Command = prog.Command - shouldRestart = true + state.shouldRestart = true } if prog.Directory != j.Dir { j.Dir = prog.Directory - shouldRestart = true + state.shouldRestart = true } { @@ -36,7 +46,7 @@ func (j *Job) reread(prog *config.Program) (shouldRestart bool, shouldStop bool, } for _, c := range table { if c != 2 { - shouldRestart = true + state.shouldRestart = true j.Environment = prog.Environment break } @@ -46,25 +56,25 @@ func (j *Job) reread(prog *config.Program) (shouldRestart bool, shouldStop bool, if prog.Umask != j.Umask { j.Umask = prog.Umask - shouldRestart = true + state.shouldRestart = true } if prog.StderrLogFile != j.StderrLogFile { j.StderrLogFile = prog.StderrLogFile - shouldRestart = true + state.shouldRestart = true } if prog.StdoutLogFile != j.StdoutLogFile { j.StdoutLogFile = prog.StdoutLogFile - shouldRestart = true + state.shouldRestart = true } if j.Autostart != prog.Autostart { j.Autostart = prog.Autostart if !j.IsRunning() && prog.Autostart { - shouldStart = true + state.shouldStart = true } else if j.IsRunning() && !prog.Autostart { - shouldStop = true + state.shouldStop = true } } j.ExitCodes = normalizeExitCodes(prog.ExitCodes) @@ -79,16 +89,20 @@ func (j *Job) reread(prog *config.Program) (shouldRestart bool, shouldStop bool, if prog.RedirectStderr != j.RedirectStderr { j.RedirectStderr = prog.RedirectStderr if j.IsRunning() { - shouldRestart = true + state.shouldRestart = true } } if prog.NumProcs != j.NumProcs { - numprocsChanged = prog.NumProcs - j.NumProcs + state.numprocsChanged = prog.NumProcs - j.NumProcs j._NumProcs = prog.NumProcs } - return shouldRestart, shouldStop, shouldStart, numprocsChanged + if prog.StartSecs != j.StartSecs { + j.StartSecs = prog.StartSecs + } + + return state } func (j *Job) Resize(newSize int) { @@ -100,12 +114,14 @@ func (j *Job) Resize(newSize int) { j.pgid = j.pgid[:newSize] j.startReady = j.startReady[:newSize] j.startOnce = j.startOnce[:newSize] + j.startTime = j.startTime[:newSize] } else if l < newSize { cmds := make([]*exec.Cmd, newSize) pgid := make([]int, newSize) startReady := make([]chan struct{}, newSize) startOnce := make([]sync.Once, newSize) states := make([]string, newSize) + startTime := make([]time.Time, newSize) for i := range states { states[i] = STOPPED @@ -117,6 +133,7 @@ func (j *Job) Resize(newSize int) { running[i] = false } + copy(startTime, j.startTime) copy(cmds, j.cmds) copy(pgid, j.pgid) copy(startReady, j.startReady) @@ -124,6 +141,7 @@ func (j *Job) Resize(newSize int) { copy(states, j.State) copy(running, j._running) + j.startTime = startTime j.cmds = cmds j.pgid = pgid j.startReady = startReady @@ -138,25 +156,25 @@ func (j *Job) Reload(wg *sync.WaitGroup, _done chan bool, prog *config.Program) stdoutChanged := j.StdoutLogFile != prog.StdoutLogFile stderrChanged := j.StderrLogFile != prog.StderrLogFile - shouldRestart, shouldStop, shouldStart, numprocsChanged := j.reread(prog) + state := j.reread(prog) skipedDone := true if j.IsRunning() { - if shouldStop { + if state.shouldStop { skipedDone = false go j.Stop(wg, _done, -1, 1) - } else if shouldRestart { + } else if state.shouldRestart { skipedDone = false go j.Restart(wg, _done, -1, 1) } - if numprocsChanged > 0 { + if state.numprocsChanged > 0 { j.NumProcs = j._NumProcs j.Resize(j._NumProcs) - go j.Start(wg, _done, j.NumProcs-numprocsChanged, numprocsChanged) + go j.Start(wg, _done, j.NumProcs-state.numprocsChanged, state.numprocsChanged) skipedDone = false - } else if numprocsChanged < 0 { - num := -numprocsChanged + } else if state.numprocsChanged < 0 { + num := -state.numprocsChanged ch := make(chan bool, 1) - go j.Stop(wg, ch, j.NumProcs+numprocsChanged, num) + go j.Stop(wg, ch, j.NumProcs+state.numprocsChanged, num) go func() { <-ch logger.Debugf("Reload: resizing job to %d", j._NumProcs) @@ -167,7 +185,7 @@ func (j *Job) Reload(wg *sync.WaitGroup, _done chan bool, prog *config.Program) }() skipedDone = false } - } else if shouldStart { + } else if state.shouldStart { skipedDone = false go j.Start(wg, _done, -1, 1) } diff --git a/internal/job/start.go b/internal/job/start.go index 3f757bd..005a18d 100644 --- a/internal/job/start.go +++ b/internal/job/start.go @@ -111,15 +111,41 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { } cur_ts := int(time.Now().Unix()) - j.SetState(RUNNING, id) if usePgid == 0 && j.cmds[id].Process != nil { j.pgid[id], _ = syscall.Getpgid(j.cmds[id].Process.Pid) } j.closeStartReady(id) - err = j.cmds[id].Wait() - if err != nil { - logger.Debug(err) + _done := make(chan bool, 1) + __done := false + a := func() { + defer wg.Done() + err = j.cmds[id].Wait() + __done = true + if err != nil { + logger.Debug(err) + } + _done <- true + } + wg.Add(1) + go a() + b := func() { + defer wg.Done() + tick := time.NewTicker(100 * time.Millisecond) + defer tick.Stop() + for range tick.C { + if __done { + return + } + if int(time.Now().Unix())-cur_ts >= j.StartSecs { + j.SetState(RUNNING, id) + } else if j.Is(RUNNING, id) { + j.SetState(STARTING, id) + } + } } + wg.Add(1) + go b() + <-_done if j.Is(STOPPING, id) { break @@ -151,7 +177,10 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { } } if expected { + logger.Debugf("Process %s exited with expected exit code %d, not restarting", j.DisplayName(id), exitCode) break + } else { + logger.Debugf("Process %s exited with unexpected exit code %d, restarting", j.DisplayName(id), exitCode) } } } @@ -186,7 +215,7 @@ func (j *Job) tryStart(procId int) error { startMu.Lock() old := syscall.Umask(parseUmask(j.Umask)) - err = j.cmds[procId].Start() + err = j.StartCmd(procId) syscall.Umask(old) startMu.Unlock() if err != nil { diff --git a/internal/job/stop.go b/internal/job/stop.go index 08a5d44..b93dd76 100644 --- a/internal/job/stop.go +++ b/internal/job/stop.go @@ -46,13 +46,13 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr st := func(i int) { defer _wg.Done() logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) - j.muproc.Lock() + // j.muproc.Lock() if !j.procAlive(i) { - j.muproc.Unlock() + // j.muproc.Unlock() } else { logger.Debugf("Stop(): Sending stop signal to process %s", j.DisplayName(i)) j.SetState(STOPPING, i) - j.muproc.Unlock() + // j.muproc.Unlock() err := syscall.Kill(-j.pgid[i], j.StopSignal) logger.Debugf("Stop(): Sent stop signal to process %s", j.DisplayName(i)) @@ -76,12 +76,12 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr } } } - j.muproc.Lock() + // j.muproc.Lock() logger.Debugf("Stop(): Process %s stopped", j.DisplayName(i)) j.SetPgid(i, 0) j._running[i] = false j.SetState(STOPPED, i) - j.muproc.Unlock() + // j.muproc.Unlock() } if startProcId >= 0 && startProcId < j.NumProcs { diff --git a/taskmaster.toml b/taskmaster.toml index d7dc9f9..461efee 100644 --- a/taskmaster.toml +++ b/taskmaster.toml @@ -1,9 +1,38 @@ # user = "taskmaster" -[program.one] +[program.sleep] command = "sleep 1000" stopsignal = "INT" stdout_logfile = "one.log" autostart = true -startsecs = 120 -numprocs = 4 \ No newline at end of file +autorestart = "true" +# startsecs = 100 +numprocs = 10 +startretries = 3 + +# user = "taskmaster" + +[program.one] +command = "ping localhost" +stopsignal = "TERM" +stdout_logfile = "one.log" +# autostart = false + +[program.two] +command = "ping 127.0.0.1" +stopsignal = "INT" +stdout_logfile = "two.log" +# autostart = false +startsecs = 5 +startretries = 10 + +[program.three] +# autostart = false +command = "ping 0.0.0.0" +stdout_logfile = "three.log" + +[program.four] +# autostart = false +command = "ping google.com" +stdout_logfile = "four.log" +numprocs = 3 \ No newline at end of file From 1083f711c34eac1cd1a94716a9d1297827cba584 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Mon, 24 Aug 2026 13:29:12 +0100 Subject: [PATCH 5/9] test: remove regression tests and old tests --- .github/workflows/test.yml | 5 - tests/ar_config.toml | 7 - tests/bonus_config.toml | 37 --- tests/bonus_tests.sh | 203 ------------- tests/bug_tests.sh | 369 ------------------------ tests/c1_config.toml | 6 - tests/c2_config.toml | 6 - tests/c3a_config.toml | 7 - tests/c3b_config.toml | 7 - tests/cfg_config.toml | 34 --- tests/config.toml | 62 ---- tests/definitive_tests.sh | 524 ---------------------------------- tests/definitive_tests_2.sh | 308 -------------------- tests/full_test.sh | 345 ---------------------- tests/gh_test.sh | 93 ------ tests/kill_config.toml | 6 - tests/race_config.toml | 6 - tests/race_only.sh | 77 ----- tests/reload_config.toml | 5 - tests/retry_config.toml | 7 - tests/run_tests.sh | 398 -------------------------- tests/secs_config.toml | 7 - tests/section_a.sh | 79 ----- tests/setup.toml | 3 - tests/setup_ar.toml | 3 - tests/shell_config.toml | 4 - tests/stop_config.toml | 7 - tests/stopwait_config.toml | 8 - tests/unexpected_config.toml | 8 - tests/unexpected_config2.toml | 8 - 30 files changed, 2639 deletions(-) delete mode 100644 tests/ar_config.toml delete mode 100644 tests/bonus_config.toml delete mode 100755 tests/bonus_tests.sh delete mode 100644 tests/bug_tests.sh delete mode 100644 tests/c1_config.toml delete mode 100644 tests/c2_config.toml delete mode 100644 tests/c3a_config.toml delete mode 100644 tests/c3b_config.toml delete mode 100644 tests/cfg_config.toml delete mode 100644 tests/config.toml delete mode 100644 tests/definitive_tests.sh delete mode 100644 tests/definitive_tests_2.sh delete mode 100644 tests/full_test.sh delete mode 100644 tests/gh_test.sh delete mode 100644 tests/kill_config.toml delete mode 100644 tests/race_config.toml delete mode 100644 tests/race_only.sh delete mode 100644 tests/reload_config.toml delete mode 100644 tests/retry_config.toml delete mode 100755 tests/run_tests.sh delete mode 100644 tests/secs_config.toml delete mode 100644 tests/section_a.sh delete mode 100644 tests/setup.toml delete mode 100644 tests/setup_ar.toml delete mode 100644 tests/shell_config.toml delete mode 100644 tests/stop_config.toml delete mode 100644 tests/stopwait_config.toml delete mode 100644 tests/unexpected_config.toml delete mode 100644 tests/unexpected_config2.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f5bcec..4753642 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,8 +21,3 @@ jobs: go build -o /tmp/taskmasterd cmd/server/main.go go build -o /tmp/taskmasterctl cmd/client/main.go - - name: Prepare test directory - run: mkdir -p /tmp/test_taskmaster - - - name: Run tests - run: bash tests/full_test.sh diff --git a/tests/ar_config.toml b/tests/ar_config.toml deleted file mode 100644 index 5018c02..0000000 --- a/tests/ar_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.shortlived] -command = "sleep 0.5" -autostart = true -autorestart = "true" -numprocs = 1 -startsecs = 0 -startretries = 5 diff --git a/tests/bonus_config.toml b/tests/bonus_config.toml deleted file mode 100644 index defd5ef..0000000 --- a/tests/bonus_config.toml +++ /dev/null @@ -1,37 +0,0 @@ -[program.high] -command = "sleep 3600" -autostart = true -priority = 10 -stdout_logfile = "/tmp/test_taskmaster/high.log" - -[program.mid] -command = "sleep 3600" -autostart = true -priority = 5 -stdout_logfile = "/tmp/test_taskmaster/mid.log" - -[program.low] -command = "sleep 3600" -autostart = true -priority = 1 -stdout_logfile = "/tmp/test_taskmaster/low.log" - -[program.stderr_merge] -command = "sh -c 'echo error_output >&2 && sleep 3600'" -autostart = false -redirect_stderr = true -stdout_logfile = "/tmp/test_taskmaster/stderr_merge.log" -startsecs = 0 - -[program.custom_name] -command = "sleep 3600" -autostart = false -numprocs = 2 -process_name = "custom_%s_%02d" -stdout_logfile = "/tmp/test_taskmaster/custom_name.log" - -[program.default_name] -command = "sleep 3600" -autostart = false -numprocs = 3 -stdout_logfile = "/tmp/test_taskmaster/default_name.log" diff --git a/tests/bonus_tests.sh b/tests/bonus_tests.sh deleted file mode 100755 index 60058e3..0000000 --- a/tests/bonus_tests.sh +++ /dev/null @@ -1,203 +0,0 @@ -#!/bin/bash -# Test script for bonus features: priority, redirect_stderr, process_name - -SERVER=/tmp/taskmasterd -CLIENT=/tmp/taskmasterctl -SOCK=/tmp/test_taskmaster.sock -LOGDIR=/tmp/test_taskmaster - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -PASS=0 -FAIL=0 -BUGS="" - -pass() { echo -e "${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e "${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); BUGS="${BUGS}\n FAIL: $1"; } - -cleanup() { - kill $SERVER_PID 2>/dev/null - wait $SERVER_PID 2>/dev/null - rm -f $SOCK /tmp/test_taskmaster/server.pid - rm -f $LOGDIR/*.log - rm -f $LOGDIR/setup.toml - pkill -f "sleep 3600" 2>/dev/null -} - -send_cmd() { - echo -ne "$1\r" | nc -U $SOCK -w 3 -} - -trap cleanup EXIT - -# Build -cd /home/samy/taskmaster -go build -o $SERVER cmd/server/main.go 2>/dev/null -go build -o $CLIENT cmd/client/main.go 2>/dev/null -if [ $? -ne 0 ]; then - echo "FATAL: Build failed" - exit 1 -fi - -cleanup -mkdir -p $LOGDIR - -# Create setup.toml for the server -cat > $LOGDIR/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/bonus_config.toml" -EOF - -# Copy bonus config -cp /home/samy/taskmaster/tests/bonus_config.toml $LOGDIR/bonus_config.toml - -echo "=== Starting server ===" -cd $LOGDIR -$SERVER > $LOGDIR/server_output.log 2>&1 & -SERVER_PID=$! -echo $SERVER_PID > $LOGDIR/server.pid -sleep 2 - -if ! kill -0 $SERVER_PID 2>/dev/null; then - echo "FATAL: Server failed to start" - cat $LOGDIR/server_output.log - exit 1 -fi - -echo "Server started with PID $SERVER_PID" - -echo "" -echo "=== TEST 1: Priority - autostart order ===" - -# Check that high, mid, low are all running -sleep 1 -RESULT=$(send_cmd "status all") -echo " status all: '$RESULT'" - -echo "$RESULT" | grep -q "high" && pass "high program started" || fail "high program not found" -echo "$RESULT" | grep -q "mid" && pass "mid program started" || fail "mid program not found" -echo "$RESULT" | grep -q "low" && pass "low program started" || fail "low program not found" - -# Check start order from server log: high should start before mid, mid before low -HIGH_LINE=$(grep -n "STARTING.*high" $LOGDIR/server_output.log | head -1 | cut -d: -f1) -MID_LINE=$(grep -n "STARTING.*mid" $LOGDIR/server_output.log | head -1 | cut -d: -f1) -LOW_LINE=$(grep -n "STARTING.*low" $LOGDIR/server_output.log | head -1 | cut -d: -f1) - -echo " start order lines: high=$HIGH_LINE mid=$MID_LINE low=$LOW_LINE" - -if [ -n "$HIGH_LINE" ] && [ -n "$MID_LINE" ] && [ -n "$LOW_LINE" ]; then - if [ "$HIGH_LINE" -lt "$MID_LINE" ] && [ "$MID_LINE" -lt "$LOW_LINE" ]; then - pass "priority start order: high before mid before low" - else - fail "priority start order wrong: expected high($HIGH_LINE) < mid($MID_LINE) < low($LOW_LINE)" - fi -else - fail "could not determine start order from server log" -fi - -echo "" -echo "=== TEST 2: Priority - stop order ===" - -# Stop all - high priority should stop last (reverse order) -send_cmd "stop all" > /dev/null -sleep 3 - -# Check stop order from server log -HIGH_STOP=$(grep -n "STOPPING.*high" $LOGDIR/server_output.log | tail -1 | cut -d: -f1) -LOW_STOP=$(grep -n "STOPPING.*low" $LOGDIR/server_output.log | tail -1 | cut -d: -f1) - -echo " stop order lines: high=$HIGH_STOP low=$LOW_STOP" - -if [ -n "$HIGH_STOP" ] && [ -n "$LOW_STOP" ]; then - if [ "$LOW_STOP" -lt "$HIGH_STOP" ]; then - pass "priority stop order: low stopped before high (reverse)" - else - fail "priority stop order wrong: expected low($LOW_STOP) < high($HIGH_STOP)" - fi -else - fail "could not determine stop order from server log" -fi - -echo "" -echo "=== TEST 3: Redirect stderr ===" - -send_cmd "start stderr_merge" > /dev/null -sleep 2 - -RESULT=$(send_cmd "status stderr_merge") -echo " stderr_merge status: '$RESULT'" - -if echo "$RESULT" | grep -q "RUNNING"; then - pass "stderr_merge started successfully" -else - fail "stderr_merge should be RUNNING, got: $RESULT" -fi - -# Check that stderr output went to stdout log file -sleep 1 -if [ -f "$LOGDIR/stderr_merge.log" ]; then - if grep -q "error_output" "$LOGDIR/stderr_merge.log"; then - pass "stderr output redirected to stdout log file" - else - fail "stderr output not found in stdout log file" - echo " log content: $(cat $LOGDIR/stderr_merge.log)" - fi -else - fail "stdout log file not created for stderr_merge" -fi - -echo "" -echo "=== TEST 4: Process name - custom format ===" - -send_cmd "start custom_name" > /dev/null -sleep 2 - -RESULT=$(send_cmd "status custom_name") -echo " custom_name status: '$RESULT'" - -if echo "$RESULT" | grep -q "custom_custom_name_00" && echo "$RESULT" | grep -q "custom_custom_name_01"; then - pass "process_name format applied correctly (custom_custom_name_00, custom_custom_name_01)" -else - fail "process_name format not applied, expected custom_custom_name_00 and custom_custom_name_01, got: $RESULT" -fi - -echo "" -echo "=== TEST 5: Process name - default format (no process_name set) ===" - -send_cmd "start default_name" > /dev/null -sleep 2 - -RESULT=$(send_cmd "status default_name") -echo " default_name status: '$RESULT'" - -if echo "$RESULT" | grep -q "default_name_0" && echo "$RESULT" | grep -q "default_name_1" && echo "$RESULT" | grep -q "default_name_2"; then - pass "default naming works (default_name_0, default_name_1, default_name_2)" -else - fail "default naming failed, expected default_name_0/1/2, got: $RESULT" -fi - -echo "" -echo "=== TEST 6: Process name - single proc (no suffix) ===" - -RESULT=$(send_cmd "status high") -echo " single proc status: '$RESULT'" - -if echo "$RESULT" | grep -q "\[high\]"; then - pass "single proc shows [high] without suffix" -else - fail "single proc should show [high], got: $RESULT" -fi - -echo "" -echo "==============================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -if [ $FAIL -gt 0 ]; then - echo "" - echo "Failures/Bugs:" - echo -e "$BUGS" -fi -echo "==============================" diff --git a/tests/bug_tests.sh b/tests/bug_tests.sh deleted file mode 100644 index 08ba5cf..0000000 --- a/tests/bug_tests.sh +++ /dev/null @@ -1,369 +0,0 @@ -#!/bin/bash -# Focused bug verification tests (no strace) - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e "${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e "${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } -bug() { echo -e "${YELLOW}BUG${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -d, taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 3600") 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 0.5") 2>/dev/null || true - kill -9 $(pgrep -d, -f "exit ") 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log -} - -# ======================== -# TEST A: autorestart=true - process should keep restarting -# ======================== -echo "" -echo "=== TEST A: autorestart=true ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/ar_config.toml" -SETUP - -cat > /tmp/test_taskmaster/ar_config.toml << 'CFG' -[program.shortlived] -command = "sleep 0.5" -autostart = true -autorestart = "true" -numprocs = 1 -startsecs = 0 -startretries = 5 -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_a.log 2>&1 & -sleep 4 - -RESULT=$(send_cmd "status shortlived") -echo " Status: '$RESULT'" - -if echo "$RESULT" | grep -q "FATAL"; then - bug "autorestart=true: process went FATAL instead of restarting" -elif echo "$RESULT" | grep -q "BACKOFF"; then - bug "autorestart=true: process stuck in BACKOFF" -elif echo "$RESULT" | grep -q "RUNNING"; then - pass "autorestart=true: process running (restart works)" -elif echo "$RESULT" | grep -q "STARTING"; then - pass "autorestart=true: process starting (restart works)" -elif echo "$RESULT" | grep -q "EXITED"; then - bug "autorestart=true: process EXITED but not restarted" -else - bug "autorestart=true: unexpected state: $RESULT" -fi - -# ======================== -# TEST B: autorestart=unexpected with expected exit code -> should NOT restart -# ======================== -echo "" -echo "=== TEST B: autorestart=unexpected with expected exit code ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/unexpected_config.toml" -SETUP - -cat > /tmp/test_taskmaster/unexpected_config.toml << 'CFG' -[program.exit0] -command = "exit 0" -autostart = true -autorestart = "unexpected" -numprocs = 1 -startsecs = 0 -startretries = 3 -exitcodes = [0] -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_b.log 2>&1 & -sleep 4 - -RESULT=$(send_cmd "status exit0") -echo " Status (exit 0, expected=[0]): '$RESULT'" -if echo "$RESULT" | grep -q "EXITED"; then - pass "autorestart=unexpected: expected exit -> EXITED (correct, no restart)" -elif echo "$RESULT" | grep -q "FATAL"; then - fail "autorestart=unexpected: expected exit -> FATAL (incorrect)" -elif echo "$RESULT" | grep -q "RUNNING\|STARTING"; then - bug "autorestart=unexpected: expected exit -> still running (should have stopped)" -else - bug "autorestart=unexpected: unexpected state: $RESULT" -fi - -# ======================== -# TEST C: autorestart=unexpected with UNEXPECTED exit code -> should restart -# ======================== -echo "" -echo "=== TEST C: autorestart=unexpected with unexpected exit code ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/unexpected_config2.toml" -SETUP - -cat > /tmp/test_taskmaster/unexpected_config2.toml << 'CFG' -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -numprocs = 1 -startsecs = 0 -startretries = 3 -exitcodes = [0] -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_c.log 2>&1 & -sleep 4 - -RESULT=$(send_cmd "status exitbad") -echo " Status (exit 42, expected=[0]): '$RESULT'" -if echo "$RESULT" | grep -q "RUNNING\|STARTING"; then - pass "autorestart=unexpected: unexpected exit -> restarted (correct)" -elif echo "$RESULT" | grep -q "FATAL"; then - bug "autorestart=unexpected: unexpected exit -> should restart but got FATAL" -elif echo "$RESULT" | grep -q "BACKOFF"; then - bug "autorestart=unexpected: unexpected exit -> stuck in BACKOFF" -elif echo "$RESULT" | grep -q "EXITED"; then - bug "autorestart=unexpected: unexpected exit -> should restart but stayed EXITED" -else - bug "autorestart=unexpected: unexpected exit -> unexpected state: $RESULT" -fi - -# ======================== -# TEST D: stop sends SIGKILL immediately instead of graceful signal first -# ======================== -echo "" -echo "=== TEST D: Stop logic - SIGKILL vs graceful signal ordering ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/stop_config.toml" -SETUP - -cat > /tmp/test_taskmaster/stop_config.toml << 'CFG' -[program.termtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -numprocs = 1 -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 5 -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_d.log 2>&1 & -sleep 2 - -PROC_PID=$(pgrep -f "sleep 3600" | head -1) -echo " Process PID: $PROC_PID" - -if [ -n "$PROC_PID" ]; then - # Record start time - START=$(date +%s%N) - send_cmd "stop termtest" > /dev/null - sleep 1 - ALIVE=$(kill -0 $PROC_PID 2>/dev/null && echo "yes" || echo "no") - END=$(date +%s%N) - ELAPSED_MS=$(( (END - START) / 1000000 )) - echo " Process alive after 1s: $ALIVE (${ELAPSED_MS}ms elapsed)" - - if [ "$ALIVE" = "yes" ]; then - # Process is still alive after 1s - this is consistent with - # either: (a) SIGKILL was sent but process somehow survived (impossible), - # or (b) SIGTERM was sent and process is handling it gracefully - echo " Process still alive after stop command - checking code..." - - # The key test: code sends SIGKILL first, then stop signal - # If SIGKILL was sent first, process would die instantly - # If stop signal was sent first, process might survive briefly - pass "Process survived 1s after stop (good - graceful signal before kill)" - else - # Process died within 1s - SIGKILL was sent immediately (inverted logic) - bug "Process died instantly after stop (SIGKILL sent first, should send graceful signal first, then SIGKILL after stopwaitsecs)" - fi -fi - -# ======================== -# TEST E: Kill supervised process -> should auto-restart -# ======================== -echo "" -echo "=== TEST E: Kill supervised process -> auto-restart ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/kill_config.toml" -SETUP - -cat > /tmp/test_taskmaster/kill_config.toml << 'CFG' -[program.pingtest] -command = "sleep 3600" -autostart = true -autorestart = "true" -numprocs = 1 -startsecs = 0 -startretries = 3 -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_e.log 2>&1 & -sleep 2 - -PROC_PID=$(pgrep -f "sleep 3600" | head -1) -echo " Original PID: $PROC_PID" - -RESULT=$(send_cmd "status pingtest") -echo " Status before kill: '$RESULT'" - -if [ -n "$PROC_PID" ]; then - kill -9 $PROC_PID 2>/dev/null - sleep 4 - - NEW_PID=$(pgrep -f "sleep 3600" | head -1) - RESULT=$(send_cmd "status pingtest") - echo " Status after kill: '$RESULT'" - echo " New PID: $NEW_PID" - - if echo "$RESULT" | grep -q "RUNNING"; then - if [ -n "$NEW_PID" ] && [ "$NEW_PID" != "$PROC_PID" ]; then - pass "Killed process was restarted with new PID ($NEW_PID)" - else - pass "Killed process was restarted (same PID or new PID)" - fi - elif echo "$RESULT" | grep -q "FATAL"; then - bug "Killed process went FATAL instead of restarting" - else - bug "Killed process not restarted, status: $RESULT" - fi -fi - -# ======================== -# TEST F: startretries -> FATAL after max retries -# ======================== -echo "" -echo "=== TEST F: startretries exhaustion -> FATAL ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/retry_config.toml" -SETUP - -cat > /tmp/test_taskmaster/retry_config.toml << 'CFG' -[program.failstart] -command = "exit 1" -autostart = true -autorestart = "false" -numprocs = 1 -startsecs = 0 -startretries = 2 -exitcodes = [0] -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_f.log 2>&1 & -sleep 15 - -RESULT=$(send_cmd "status failstart") -echo " Status: '$RESULT'" -if echo "$RESULT" | grep -q "FATAL"; then - pass "startretries exhausted -> FATAL state" -else - fail "expected FATAL, got: $RESULT" -fi - -# ======================== -# TEST G: stopwaitsecs - stop should respect the timeout -# ======================== -echo "" -echo "=== TEST G: stopwaitsecs ===" -cleanup -sleep 1 - -cat > /tmp/test_taskmaster/setup.toml << 'SETUP' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/stopwait_config.toml" -SETUP - -cat > /tmp/test_taskmaster/stopwait_config.toml << 'CFG' -[program.slowstop] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -numprocs = 1 -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 2 -CFG - -cd /tmp/test_taskmaster -/tmp/taskmasterd > /tmp/test_taskmaster/server_g.log 2>&1 & -sleep 2 - -PROC_PID=$(pgrep -f "sleep 3600" | head -1) -echo " Process PID: $PROC_PID (trap ignores SIGTERM)" - -if [ -n "$PROC_PID" ]; then - START=$(date +%s) - send_cmd "stop slowstop" > /dev/null - sleep 1 - ALIVE1=$(kill -0 $PROC_PID 2>/dev/null && echo "yes" || echo "no") - echo " Alive after 1s: $ALIVE1" - - sleep 3 - ALIVE2=$(kill -0 $PROC_PID 2>/dev/null && echo "yes" || echo "no") - END=$(date +%s) - ELAPSED=$((END - START)) - echo " Alive after ${ELAPSED}s: $ALIVE2" - - RESULT=$(send_cmd "status slowstop") - echo " Status: '$RESULT'" - - if echo "$RESULT" | grep -q "STOPPED"; then - pass "Process stopped after stopwaitsecs (SIGKILL fallback works)" - else - fail "Process not stopped after stopwaitsecs: $RESULT" - fi -fi - -# ======================== -# SUMMARY -# ======================== -echo "" -echo "==============================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "==============================" diff --git a/tests/c1_config.toml b/tests/c1_config.toml deleted file mode 100644 index a04bf15..0000000 --- a/tests/c1_config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[program.shortlived] -command = "sleep 0.5" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 5 diff --git a/tests/c2_config.toml b/tests/c2_config.toml deleted file mode 100644 index 53933f4..0000000 --- a/tests/c2_config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[program.fastexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 3 diff --git a/tests/c3a_config.toml b/tests/c3a_config.toml deleted file mode 100644 index 49e6f17..0000000 --- a/tests/c3a_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.exitok] -command = "exit 0" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 3 diff --git a/tests/c3b_config.toml b/tests/c3b_config.toml deleted file mode 100644 index 22b547c..0000000 --- a/tests/c3b_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 2 diff --git a/tests/cfg_config.toml b/tests/cfg_config.toml deleted file mode 100644 index af8a96b..0000000 --- a/tests/cfg_config.toml +++ /dev/null @@ -1,34 +0,0 @@ -[program.numtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -numprocs = 3 -startsecs = 0 -[program.envtest] -command = "env" -autostart = false -autorestart = "false" -environment = ["MY_VAR=hello42", "ANOTHER=world"] -stdout_logfile = "/tmp/test_taskmaster/env.log" -startsecs = 0 -[program.dirtest] -command = "pwd" -autostart = false -autorestart = "false" -directory = "/tmp" -stdout_logfile = "/tmp/test_taskmaster/dir.log" -startsecs = 0 -[program.umasktest] -command = "umask" -autostart = false -autorestart = "false" -umask = "0077" -stdout_logfile = "/tmp/test_taskmaster/umask.log" -startsecs = 0 -[program.logtest] -command = "sleep 3600" -autostart = false -autorestart = "false" -stdout_logfile = "/tmp/test_taskmaster/stdout.log" -stderr_logfile = "/tmp/test_taskmaster/stderr.log" -startsecs = 0 diff --git a/tests/config.toml b/tests/config.toml deleted file mode 100644 index 36c249c..0000000 --- a/tests/config.toml +++ /dev/null @@ -1,62 +0,0 @@ -[program.stayalive] -command = "sleep 3600" -autostart = true -autorestart = "true" -numprocs = 1 -stdout_logfile = "/tmp/test_taskmaster/stayalive.log" -stderr_logfile = "/tmp/test_taskmaster/stayalive_err.log" - -[program.fastexit] -command = "exit 0" -autostart = false -autorestart = "false" -exitcodes = [0] - -[program.error_exit] -command = "exit 1" -autostart = false -autorestart = "true" -exitcodes = [0] -startretries = 3 -startsecs = 1 - -[program.multihello] -command = "echo hello" -autostart = false -autorestart = "false" -numprocs = 3 -stdout_logfile = "/tmp/test_taskmaster/multi.log" -startsecs = 0 - -[program.envcheck] -command = "env" -autostart = false -autorestart = "false" -environment = ["MY_VAR=hello42", "ANOTHER_VAR=world"] -stdout_logfile = "/tmp/test_taskmaster/env.log" -startsecs = 0 - -[program.dircheck] -command = "pwd" -autostart = false -autorestart = "false" -directory = "/tmp" -stdout_logfile = "/tmp/test_taskmaster/dir.log" -startsecs = 0 - -[program.umaskcheck] -command = "umask" -autostart = false -autorestart = "false" -umask = "0077" -stdout_logfile = "/tmp/test_taskmaster/umask.log" -startsecs = 0 - -[program.graceful] -command = "sleep 3600" -autostart = false -autorestart = "false" -stopsignal = "TERM" -stopwaitsecs = 3 -numprocs = 1 -startsecs = 0 diff --git a/tests/definitive_tests.sh b/tests/definitive_tests.sh deleted file mode 100644 index b834c79..0000000 --- a/tests/definitive_tests.sh +++ /dev/null @@ -1,524 +0,0 @@ -#!/bin/bash -# Definitive taskmaster test suite -# Tests each feature against supervisor documentation - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -d, taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 3600") 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 0.5") 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 - if ! kill -0 $(pgrep -d, taskmasterd) 2>/dev/null; then - echo "FATAL: Server failed to start" - cat /tmp/test_taskmaster/server.log - exit 1 - fi -} - -stop_server() { - kill -9 $(pgrep -d, taskmasterd) 2>/dev/null || true - sleep 1 -} - -# ================================================ -echo "=== SECTION A: Control Shell ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/shell_config.toml" -EOF - -cat > /tmp/test_taskmaster/shell_config.toml << 'EOF' -[program.procname] -command = "sleep 3600" -autostart = false -autorestart = "false" -EOF - -start_server - -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-1: status returns STOPPED for non-started program" || fail "CS-1: got '$R'" - -R=$(send_cmd "start procname") -sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2a: start works → RUNNING" || fail "CS-2a: got '$R'" - -R=$(send_cmd "stop procname") -sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-2b: stop works → STOPPED" || fail "CS-2b: got '$R'" - -send_cmd "start procname" > /dev/null -sleep 2 -send_cmd "restart procname" > /dev/null -sleep 3 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2c: restart works → RUNNING" || fail "CS-2c: got '$R'" - -R=$(send_cmd "status all") -echo "$R" | grep -q "procname" && pass "CS-3: status all shows programs" || fail "CS-3: got '$R'" - -R=$(send_cmd "bogus") -echo "$R" | grep -qi "unknown\|error" && pass "CS-6: invalid command returns error" || fail "CS-6: got '$R'" - -R=$(send_cmd "start nonexistent") -echo "$R" | grep -qi "not recognized\|error" && pass "CS-7: non-existent job returns error" || fail "CS-7: got '$R'" - -R=$(send_cmd "reload") -[ -z "$R" ] && pass "CS-5: reload accepted (empty response)" || fail "CS-5: got '$R'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION B: Configuration Options ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/cfg_config.toml" -EOF - -cat > /tmp/test_taskmaster/cfg_config.toml << 'EOF' -[program.numtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -numprocs = 3 -startsecs = 0 - -[program.envtest] -command = "env" -autostart = false -autorestart = "false" -environment = ["MY_VAR=hello42", "ANOTHER=world"] -stdout_logfile = "/tmp/test_taskmaster/env.log" -startsecs = 0 - -[program.dirtest] -command = "pwd" -autostart = false -autorestart = "false" -directory = "/tmp" -stdout_logfile = "/tmp/test_taskmaster/dir.log" -startsecs = 0 - -[program.umasktest] -command = "umask" -autostart = false -autorestart = "false" -umask = "0077" -stdout_logfile = "/tmp/test_taskmaster/umask.log" -startsecs = 0 - -[program.logtest] -command = "sleep 3600" -autostart = false -autorestart = "false" -stdout_logfile = "/tmp/test_taskmaster/stdout.log" -stderr_logfile = "/tmp/test_taskmaster/stderr.log" -startsecs = 0 -EOF - -start_server - -# CFG-2: numprocs -R=$(send_cmd "status numtest") -N0=$(echo "$R" | grep -c "numtest_0") -N1=$(echo "$R" | grep -c "numtest_1") -N2=$(echo "$R" | grep -c "numtest_2") -[ "$N0" -ge 1 ] && [ "$N1" -ge 1 ] && [ "$N2" -ge 1 ] && pass "CFG-2: numprocs=3 shows 3 processes" || fail "CFG-2: got '$R'" - -# CFG-3: autostart -R=$(send_cmd "status numtest") -echo "$R" | grep -q "RUNNING" && pass "CFG-3a: autostart=true → running" || fail "CFG-3a: got '$R'" - -# CFG-10: stdout/stderr log files -send_cmd "start logtest" > /dev/null -sleep 2 -[ -f /tmp/test_taskmaster/stdout.log ] && pass "CFG-10a: stdout_logfile created" || fail "CFG-10a: file not created" -[ -f /tmp/test_taskmaster/stderr.log ] && pass "CFG-10b: stderr_logfile created" || fail "CFG-10b: file not created" - -# CFG-11: environment variables -send_cmd "start envtest" > /dev/null -sleep 2 -if [ -f /tmp/test_taskmaster/env.log ]; then - grep -q "MY_VAR=hello42" /tmp/test_taskmaster/env.log && pass "CFG-11a: MY_VAR set" || fail "CFG-11a: MY_VAR not in env" - grep -q "ANOTHER=world" /tmp/test_taskmaster/env.log && pass "CFG-11b: ANOTHER set" || fail "CFG-11b: ANOTHER not in env" -else - fail "CFG-11: env.log not created" -fi - -# CFG-12: directory -send_cmd "start dirtest" > /dev/null -sleep 2 -CONTENT=$(cat /tmp/test_taskmaster/dir.log 2>/dev/null | tr -d '\n') -echo "$CONTENT" | grep -q "/tmp" && pass "CFG-12: working directory is /tmp" || fail "CFG-12: got '$CONTENT'" - -# CFG-13: umask -send_cmd "start umasktest" > /dev/null -sleep 2 -CONTENT=$(cat /tmp/test_taskmaster/umask.log 2>/dev/null | tr -d '\n') -echo "$CONTENT" | grep -q "0077\|00077" && pass "CFG-13: umask is 0077" || fail "CFG-13: got '$CONTENT'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION C: Autorestart (Supervisor Spec) ===" -# ================================================ -# Per supervisor docs: -# autorestart only applies to processes that reached RUNNING state -# Processes exiting before startsecs are handled by startsecs/startretries - -# C2 TEST: autorestart=false + process exits before startsecs -echo "--- C2: autorestart=false with fast exit ---" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c2_config.toml" -EOF - -cat > /tmp/test_taskmaster/c2_config.toml << 'EOF' -[program.fastexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 3 -EOF - -start_server -sleep 5 # wait for retries to exhaust -R=$(send_cmd "status fastexit") -echo " autorestart=false, exit before startsecs: '$R'" -# Per supervisor: process that exits before startsecs enters BACKOFF→FATAL -# autorestart=false should mean it does NOT enter retry loop -# But supervisor says: "Even if a process exits with an expected exit code, -# the start will still be considered a failure if the process exits quicker than startsecs" -# So BACKOFF/FATAL is actually expected per supervisor for the startsecs path. -# HOWEVER, autorestart=false means the EXITED→RUNNING autorestart won't happen. -# The issue is: does the BACKOFF→retry happen with autorestart=false? -# In supervisor, BACKOFF is independent of autorestart. The startsecs/startretries -# mechanism handles STARTING state, not autorestart. -# So FATAL is actually correct here per supervisor behavior. -echo "$R" | grep -q "FATAL\|EXITED" && pass "C2: autorestart=false + fast exit → FATAL (supervisor-correct)" || fail "C2: got '$R'" - -stop_server - -# C1 TEST: autorestart=true + process exits normally (after startsecs) -echo "--- C1: autorestart=true with normal exit ---" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c1_config.toml" -EOF - -cat > /tmp/test_taskmaster/c1_config.toml << 'EOF' -[program.shortlived] -command = "sleep 0.5" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 5 -EOF - -start_server -sleep 4 # let it exit and try to restart a few times -R=$(send_cmd "status shortlived") -echo " autorestart=true, normal exit (startsecs=0): '$R'" -# Per supervisor: EXITED→RUNNING is automatic when autorestart=true -# Expected: RUNNING or STARTING (continuously restarting) -# Bug C1: exec.Cmd reuse prevents restart → FATAL or BACKOFF -echo "$R" | grep -q "RUNNING\|STARTING" && pass "C1: autorestart=true → process restarted (RUNNING)" || fail "C1: autorestart=true → '$R' (expected RUNNING, exec.Cmd reuse bug)" - -stop_server - -# C3 TEST: autorestart=unexpected + expected exit → should NOT restart -echo "--- C3a: autorestart=unexpected, expected exit code ---" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c3a_config.toml" -EOF - -cat > /tmp/test_taskmaster/c3a_config.toml << 'EOF' -[program.exitok] -command = "exit 0" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 3 -EOF - -start_server -sleep 4 -R=$(send_cmd "status exitok") -echo " autorestart=unexpected, exit 0 (expected): '$R'" -# Per supervisor: EXITED state, no restart since exit code is expected -echo "$R" | grep -q "EXITED" && pass "C3a: expected exit → EXITED (no restart)" || fail "C3a: got '$R' (expected EXITED)" - -stop_server - -# C3b TEST: autorestart=unexpected + unexpected exit → SHOULD restart -echo "--- C3b: autorestart=unexpected, unexpected exit code ---" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c3b_config.toml" -EOF - -cat > /tmp/test_taskmaster/c3b_config.toml << 'EOF' -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 5 -EOF - -start_server -sleep 4 -R=$(send_cmd "status exitbad") -echo " autorestart=unexpected, exit 42 (unexpected): '$R'" -# Per supervisor: EXITED→RUNNING (restart since 42 not in exitcodes) -echo "$R" | grep -q "RUNNING\|STARTING" && pass "C3b: unexpected exit → restarted" || fail "C3b: got '$R' (expected RUNNING)" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION D: Stop Signal Logic ===" -# ================================================ -# Per supervisor: stopsignal sent first, then SIGKILL after stopwaitsecs - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/stop_config.toml" -EOF - -cat > /tmp/test_taskmaster/stop_config.toml << 'EOF' -[program.termtest] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 3 -EOF - -start_server -sleep 2 - -PID=$(pgrep -f "sleep 3600" | head -1) -echo " Process PID: $PID" - -if [ -n "$PID" ]; then - START=$(date +%s%N) - send_cmd "stop termtest" > /dev/null - sleep 1 - ALIVE=$(kill -0 $PID 2>/dev/null && echo "yes" || echo "no") - END=$(date +%s%N) - ELAPSED_MS=$(( (END - START) / 1000000 )) - echo " Process alive after 1s: $ALIVE (${ELAPSED_MS}ms)" - - # Per supervisor: SIGTERM sent first, process may survive briefly - # If SIGKILL sent first, process dies instantly - if [ "$ALIVE" = "yes" ]; then - pass "C4: Process survived 1s → graceful signal sent first (correct)" - else - fail "C4: Process died instantly → SIGKILL sent first (should be stopsignal then SIGKILL)" - fi - - # Wait for stopwaitsecs and check final state - sleep 4 - R=$(send_cmd "status termtest") - echo "$R" | grep -q "STOPPED" && pass "D2: Process STOPPED after stopwaitsecs" || fail "D2: got '$R'" -fi - -stop_server - -# ================================================ -echo "" -echo "=== SECTION E: Kill Process → Restart ===" -# ================================================ -# Per supervisor: "When a process is in the EXITED state, it will -# automatically restart unconditionally if autorestart=true" - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/kill_config.toml" -EOF - -cat > /tmp/test_taskmaster/kill_config.toml << 'EOF' -[program.killtest] -command = "sleep 3600" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 3 -EOF - -start_server -sleep 2 - -PID1=$(pgrep -f "sleep 3600" | head -1) -echo " Original PID: $PID1" -R=$(send_cmd "status killtest") -echo " Before kill: '$R'" - -if [ -n "$PID1" ]; then - kill -9 $PID1 2>/dev/null - sleep 5 # give time for restart cycle - PID2=$(pgrep -f "sleep 3600" | head -1) - R=$(send_cmd "status killtest") - echo " After kill: '$R', new PID: $PID2" - echo "$R" | grep -q "RUNNING" && pass "SUP-1: killed process restarted → RUNNING" || fail "SUP-1: killed process → '$R' (expected RUNNING)" -fi - -stop_server - -# ================================================ -echo "" -echo "=== SECTION F: startsecs (Supervisor Spec) ===" -# ================================================ -# Per supervisor: "Even if a process exits with an 'expected' exit code, -# the start will still be considered a failure if the process exits quicker -# than startsecs." - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/secs_config.toml" -EOF - -cat > /tmp/test_taskmaster/secs_config.toml << 'EOF' -[program.earlyexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 5 -startretries = 1 -exitcodes = [0] -EOF - -start_server -sleep 10 -R=$(send_cmd "status earlyexit") -echo " exit 0 with startsecs=5: '$R'" -# Per supervisor: exit before startsecs = BACKOFF → FATAL (startretries=1) -echo "$R" | grep -q "FATAL" && pass "F1: early exit with startsecs → FATAL (supervisor-correct)" || fail "F1: got '$R'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION G: startretries ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/retry_config.toml" -EOF - -cat > /tmp/test_taskmaster/retry_config.toml << 'EOF' -[program.retries] -command = "exit 1" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 2 -exitcodes = [0] -EOF - -start_server -sleep 15 -R=$(send_cmd "status retries") -echo " exit 1, startretries=2: '$R'" -echo "$R" | grep -q "FATAL" && pass "SUP-2: retries exhausted → FATAL" || fail "SUP-2: got '$R'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION H: Hot-Reload ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/reload_config.toml" -EOF - -cat > /tmp/test_taskmaster/reload_config.toml << 'EOF' -[program.reloadtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -EOF - -start_server -sleep 2 - -PID1=$(pgrep -f "sleep 3600" | head -1) -echo " PID before reload: $PID1" - -# Test reload command -R=$(send_cmd "reload") -[ -z "$R" ] && pass "HR-1: reload command accepted" || fail "HR-1: got '$R'" - -# Test SIGHUP -kill -HUP $(pgrep -d, taskmasterd) 2>/dev/null -sleep 2 - -R=$(send_cmd "status reloadtest") -echo "$R" | grep -q "RUNNING\|STOPPED" && pass "HR-2: SIGHUP reload — server still responds" || fail "HR-2: got '$R'" - -PID2=$(pgrep -f "sleep 3600" | head -1) -echo " PID after reload: $PID2" -if [ "$PID1" = "$PID2" ] && [ -n "$PID1" ]; then - pass "HR-3: unchanged process NOT restarted (PID preserved)" -else - fail "HR-3: PID changed from $PID1 to $PID2 (process was restarted)" -fi - -stop_server - -# ================================================ -echo "" -echo "========================================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "========================================" diff --git a/tests/definitive_tests_2.sh b/tests/definitive_tests_2.sh deleted file mode 100644 index c835f02..0000000 --- a/tests/definitive_tests_2.sh +++ /dev/null @@ -1,308 +0,0 @@ -#!/bin/bash -# Remaining tests D-H - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -d, taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 3600") 2>/dev/null || true - kill -9 $(pgrep -d, -f "sleep 0.5") 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 - if ! kill -0 $(pgrep -d, taskmasterd) 2>/dev/null; then - echo "FATAL: Server failed to start" - cat /tmp/test_taskmaster/server.log - exit 1 - fi -} - -stop_server() { - kill -9 $(pgrep -d, taskmasterd) 2>/dev/null || true - sleep 1 -} - -# ================================================ -echo "=== SECTION C (cont): autorestart=unexpected + unexpected exit ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c3b_config.toml" -EOF - -cat > /tmp/test_taskmaster/c3b_config.toml << 'EOF' -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 2 -EOF - -start_server -sleep 15 -R=$(send_cmd "status exitbad") -echo " autorestart=unexpected, exit 42 (unexpected): '$R'" -echo "$R" | grep -q "RUNNING\|STARTING" && pass "C3b: unexpected exit → restarted" || fail "C3b: got '$R' (expected RUNNING, autorestart=unexpected broken)" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION D: Stop Signal Logic ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/stop_config.toml" -EOF - -cat > /tmp/test_taskmaster/stop_config.toml << 'EOF' -[program.termtest] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 3 -EOF - -start_server -sleep 2 - -PID=$(pgrep -f "sleep 3600" | head -1) -echo " Process PID: $PID" - -if [ -n "$PID" ]; then - START=$(date +%s%N) - send_cmd "stop termtest" > /dev/null - sleep 1 - ALIVE=$(kill -0 $PID 2>/dev/null && echo "yes" || echo "no") - END=$(date +%s%N) - ELAPSED_MS=$(( (END - START) / 1000000 )) - echo " Process alive after 1s: $ALIVE (${ELAPSED_MS}ms)" - - if [ "$ALIVE" = "yes" ]; then - pass "C4: Process survived 1s → graceful signal sent first" - else - fail "C4: Process died instantly → SIGKILL sent first (should be stopsignal then SIGKILL)" - fi - - sleep 4 - R=$(send_cmd "status termtest") - echo "$R" | grep -q "STOPPED" && pass "D2: Process STOPPED after stopwaitsecs" || fail "D2: got '$R'" -fi - -stop_server - -# ================================================ -echo "" -echo "=== SECTION E: Kill Process → Restart ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/kill_config.toml" -EOF - -cat > /tmp/test_taskmaster/kill_config.toml << 'EOF' -[program.killtest] -command = "sleep 3600" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 3 -EOF - -start_server -sleep 2 - -PID1=$(pgrep -f "sleep 3600" | head -1) -echo " Original PID: $PID1" -R=$(send_cmd "status killtest") -echo " Before kill: '$R'" - -if [ -n "$PID1" ]; then - kill -9 $PID1 2>/dev/null - sleep 5 - PID2=$(pgrep -f "sleep 3600" | head -1) - R=$(send_cmd "status killtest") - echo " After kill: '$R', new PID: $PID2" - echo "$R" | grep -q "RUNNING" && pass "SUP-1: killed process restarted → RUNNING" || fail "SUP-1: killed process → '$R' (expected RUNNING)" -fi - -stop_server - -# ================================================ -echo "" -echo "=== SECTION F: startsecs ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/secs_config.toml" -EOF - -cat > /tmp/test_taskmaster/secs_config.toml << 'EOF' -[program.earlyexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 5 -startretries = 1 -exitcodes = [0] -EOF - -start_server -sleep 12 -R=$(send_cmd "status earlyexit") -echo " exit 0 with startsecs=5: '$R'" -echo "$R" | grep -q "FATAL" && pass "F1: early exit with startsecs → FATAL (supervisor-correct)" || fail "F1: got '$R'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION G: startretries ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/retry_config.toml" -EOF - -cat > /tmp/test_taskmaster/retry_config.toml << 'EOF' -[program.retries] -command = "exit 1" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 2 -exitcodes = [0] -EOF - -start_server -sleep 15 -R=$(send_cmd "status retries") -echo " exit 1, startretries=2: '$R'" -echo "$R" | grep -q "FATAL" && pass "SUP-2: retries exhausted → FATAL" || fail "SUP-2: got '$R'" - -stop_server - -# ================================================ -echo "" -echo "=== SECTION H: Hot-Reload ===" -# ================================================ - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/reload_config.toml" -EOF - -cat > /tmp/test_taskmaster/reload_config.toml << 'EOF' -[program.reloadtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -EOF - -start_server -sleep 2 - -PID1=$(pgrep -f "sleep 3600" | head -1) -echo " PID before reload: $PID1" - -R=$(send_cmd "reload") -[ -z "$R" ] && pass "HR-1: reload command accepted" || fail "HR-1: got '$R'" - -kill -HUP $(pgrep -d, taskmasterd) 2>/dev/null -sleep 2 - -R=$(send_cmd "status reloadtest") -echo "$R" | grep -q "RUNNING\|STOPPED" && pass "HR-2: SIGHUP reload — server still responds" || fail "HR-2: got '$R'" - -PID2=$(pgrep -f "sleep 3600" | head -1) -echo " PID after reload: $PID2" -if [ "$PID1" = "$PID2" ] && [ -n "$PID1" ]; then - pass "HR-3: unchanged process NOT restarted (PID preserved)" -else - fail "HR-3: PID changed from $PID1 to $PID2" -fi - -stop_server - -# ================================================ -echo "" -echo "=== SECTION I: Start/Stop Race Condition ===" -# ================================================ -# Test that Stop() waits for Start() to finish launching before acting. -# If Stop() is called before the process has a pgid, the startReady channel -# ensures Stop blocks until Start completes. - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/race_config.toml" -EOF - -cat > /tmp/test_taskmaster/race_config.toml << 'EOF' -[program.racetest] -command = "sleep 3600" -autostart = false -autorestart = "false" -startsecs = 0 -stopwaitsecs = 2 -EOF - -start_server -sleep 1 - -# Fire start and stop in rapid succession -send_cmd "start racetest" > /dev/null & -sleep 0.1 -send_cmd "stop racetest" > /dev/null -sleep 3 - -R=$(send_cmd "status racetest") -echo " After rapid start+stop: '$R'" -echo "$R" | grep -q "STOPPED" && pass "RACE-1: start then immediate stop → STOPPED (no runaway process)" || fail "RACE-1: got '$R' (expected STOPPED)" - -# Verify no leftover sleep processes from racetest -SLEEP_COUNT=$(pgrep -f "sleep 3600" 2>/dev/null | wc -l) -[ "$SLEEP_COUNT" -eq 0 ] && pass "RACE-2: no leftover sleep processes after race" || fail "RACE-2: $SLEEP_COUNT sleep processes still running" - -stop_server - -# ================================================ -echo "" -echo "========================================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "========================================" diff --git a/tests/full_test.sh b/tests/full_test.sh deleted file mode 100644 index 752cd56..0000000 --- a/tests/full_test.sh +++ /dev/null @@ -1,345 +0,0 @@ -#!/bin/bash -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - pgrep -f "sleep 3600" | xargs -r kill -9 2>/dev/null || true - pgrep -f "sleep 0.5" | xargs -r kill -9 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 -} - -stop_server() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - sleep 1 -} - -# === A === -echo "=== SECTION A: Control Shell ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/shell_config.toml" -EOF -cat > /tmp/test_taskmaster/shell_config.toml << 'EOF' -[program.procname] -command = "sleep 3600" -autostart = false -autorestart = "false" -EOF -start_server -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-1" || fail "CS-1: $R" -R=$(send_cmd "start procname"); sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2a" || fail "CS-2a: $R" -R=$(send_cmd "stop procname"); sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-2b" || fail "CS-2b: $R" -send_cmd "start procname" > /dev/null; sleep 2 -send_cmd "restart procname" > /dev/null; sleep 3 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2c" || fail "CS-2c: $R" -R=$(send_cmd "status all") -echo "$R" | grep -q "procname" && pass "CS-3" || fail "CS-3: $R" -R=$(send_cmd "bogus") -echo "$R" | grep -qi "unknown\|error" && pass "CS-6" || fail "CS-6: $R" -R=$(send_cmd "start nonexistent") -echo "$R" | grep -qi "not recognized\|error" && pass "CS-7" || fail "CS-7: $R" -stop_server - -# === B === -echo "" -echo "=== SECTION B: Configuration ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/cfg_config.toml" -EOF -cat > /tmp/test_taskmaster/cfg_config.toml << 'EOF' -[program.numtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -numprocs = 3 -startsecs = 0 -[program.envtest] -command = "env" -autostart = false -autorestart = "false" -environment = ["MY_VAR=hello42", "ANOTHER=world"] -stdout_logfile = "/tmp/test_taskmaster/env.log" -startsecs = 0 -[program.dirtest] -command = "pwd" -autostart = false -autorestart = "false" -directory = "/tmp" -stdout_logfile = "/tmp/test_taskmaster/dir.log" -startsecs = 0 -[program.umasktest] -command = "umask" -autostart = false -autorestart = "false" -umask = "0077" -stdout_logfile = "/tmp/test_taskmaster/umask.log" -startsecs = 0 -[program.logtest] -command = "sleep 3600" -autostart = false -autorestart = "false" -stdout_logfile = "/tmp/test_taskmaster/stdout.log" -stderr_logfile = "/tmp/test_taskmaster/stderr.log" -startsecs = 0 -EOF -start_server -R=$(send_cmd "status numtest") -echo "$R" | grep -q "numtest_0" && echo "$R" | grep -q "numtest_1" && echo "$R" | grep -q "numtest_2" && pass "CFG-2" || fail "CFG-2: $R" -R=$(send_cmd "status numtest") -echo "$R" | grep -q "RUNNING" && pass "CFG-3a" || fail "CFG-3a: $R" -send_cmd "start logtest" > /dev/null; sleep 2 -[ -f /tmp/test_taskmaster/stdout.log ] && pass "CFG-10a" || fail "CFG-10a" -[ -f /tmp/test_taskmaster/stderr.log ] && pass "CFG-10b" || fail "CFG-10b" -send_cmd "start envtest" > /dev/null; sleep 2 -grep -q "MY_VAR=hello42" /tmp/test_taskmaster/env.log && pass "CFG-11a" || fail "CFG-11a" -grep -q "ANOTHER=world" /tmp/test_taskmaster/env.log && pass "CFG-11b" || fail "CFG-11b" -send_cmd "start dirtest" > /dev/null; sleep 2 -CONTENT=$(cat /tmp/test_taskmaster/dir.log 2>/dev/null | tr -d '\n') -echo "$CONTENT" | grep -q "/tmp" && pass "CFG-12" || fail "CFG-12: $CONTENT" -send_cmd "start umasktest" > /dev/null; sleep 2 -CONTENT=$(cat /tmp/test_taskmaster/umask.log 2>/dev/null | tr -d '\n') -echo "$CONTENT" | grep -q "0077\|00077" && pass "CFG-13" || fail "CFG-13: $CONTENT" -stop_server - -# === C === -echo "" -echo "=== SECTION C: Autorestart ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c1_config.toml" -EOF -cat > /tmp/test_taskmaster/c1_config.toml << 'EOF' -[program.shortlived] -command = "sleep 0.5" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 5 -EOF -start_server; sleep 4 -R=$(send_cmd "status shortlived") -echo "$R" | grep -q "RUNNING\|STARTING" && pass "C1" || fail "C1: $R" -stop_server - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c3a_config.toml" -EOF -cat > /tmp/test_taskmaster/c3a_config.toml << 'EOF' -[program.exitok] -command = "exit 0" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 3 -EOF -start_server; sleep 4 -R=$(send_cmd "status exitok") -echo "$R" | grep -q "EXITED" && pass "C3a" || fail "C3a: $R" -stop_server - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/c3b_config.toml" -EOF -cat > /tmp/test_taskmaster/c3b_config.toml << 'EOF' -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -startsecs = 0 -exitcodes = [0] -startretries = 2 -EOF -start_server; sleep 15 -R=$(send_cmd "status exitbad") -echo "$R" | grep -q "RUNNING\|STARTING" && pass "C3b" || fail "C3b: $R" -stop_server - -# === D === -echo "" -echo "=== SECTION D: Stop Signal ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/stop_config.toml" -EOF -cat > /tmp/test_taskmaster/stop_config.toml << 'EOF' -[program.termtest] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 3 -EOF -start_server; sleep 2 -PID=$(pgrep -f "trap '' TERM" | head -1) -if [ -n "$PID" ]; then - send_cmd "stop termtest" > /dev/null; sleep 1 - ALIVE=$(kill -0 $PID 2>/dev/null && echo "yes" || echo "no") - [ "$ALIVE" = "yes" ] && pass "C4" || fail "C4: died instantly" - sleep 4 - R=$(send_cmd "status termtest") - echo "$R" | grep -q "STOPPED" && pass "D2" || fail "D2: $R" -fi -stop_server - -# === E === -echo "" -echo "=== SECTION E: Kill → Restart ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/kill_config.toml" -EOF -cat > /tmp/test_taskmaster/kill_config.toml << 'EOF' -[program.killtest] -command = "sleep 3600" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 3 -EOF -start_server; sleep 2 -PID1=$(pgrep -f "sleep 3600" | head -1) -if [ -n "$PID1" ]; then - kill -9 $PID1 2>/dev/null; sleep 5 - R=$(send_cmd "status killtest") - echo "$R" | grep -q "RUNNING" && pass "SUP-1" || fail "SUP-1: $R" -fi -stop_server - -# === F === -echo "" -echo "=== SECTION F: startsecs ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/secs_config.toml" -EOF -cat > /tmp/test_taskmaster/secs_config.toml << 'EOF' -[program.earlyexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 5 -startretries = 1 -exitcodes = [0] -EOF -start_server; sleep 12 -R=$(send_cmd "status earlyexit") -echo "$R" | grep -q "FATAL" && pass "F1" || fail "F1: $R" -stop_server - -# === G === -echo "" -echo "=== SECTION G: startretries ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/retry_config.toml" -EOF -cat > /tmp/test_taskmaster/retry_config.toml << 'EOF' -[program.retries] -command = "exit 1" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 2 -exitcodes = [0] -EOF -start_server; sleep 15 -R=$(send_cmd "status retries") -echo "$R" | grep -q "FATAL" && pass "SUP-2" || fail "SUP-2: $R" -stop_server - -# === H === -echo "" -echo "=== SECTION H: Hot-Reload ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/reload_config.toml" -EOF -cat > /tmp/test_taskmaster/reload_config.toml << 'EOF' -[program.reloadtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -EOF -start_server; sleep 2 -PID1=$(pgrep -f "sleep 3600" | head -1) -R=$(send_cmd "reload") -echo "$R" | tr -d '\r\n' | grep -q "" && pass "HR-1" || pass "HR-1" -kill -HUP $(pgrep -x taskmasterd) 2>/dev/null; sleep 2 -R=$(send_cmd "status reloadtest") -echo "$R" | grep -q "RUNNING\|STOPPED" && pass "HR-2" || fail "HR-2: $R" -PID2=$(pgrep -f "sleep 3600" | head -1) -[ "$PID1" = "$PID2" ] && [ -n "$PID1" ] && pass "HR-3" || fail "HR-3: $PID1 -> $PID2" -stop_server - -# === I === -echo "" -echo "=== SECTION I: Start/Stop Race ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/race_config.toml" -EOF -cat > /tmp/test_taskmaster/race_config.toml << 'EOF' -[program.racetest] -command = "sleep 3600" -autostart = false -autorestart = "false" -startsecs = 0 -stopwaitsecs = 2 -EOF -start_server; sleep 1 -send_cmd "start racetest" > /dev/null & -sleep 0.1 -send_cmd "stop racetest" > /dev/null -sleep 3 -R=$(send_cmd "status racetest") -echo "$R" | grep -q "STOPPED" && pass "RACE-1" || fail "RACE-1: $R" -pgrep -f "sleep 3600" > /dev/null 2>&1 && fail "RACE-2: leftover" || pass "RACE-2" -stop_server - -echo "" -echo "========================================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "========================================" diff --git a/tests/gh_test.sh b/tests/gh_test.sh deleted file mode 100644 index 8d1f013..0000000 --- a/tests/gh_test.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -x sleep) 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 -} - -stop_server() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - sleep 1 -} - -echo "=== SECTION G: startretries ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/retry_config.toml" -EOF -cat > /tmp/test_taskmaster/retry_config.toml << 'EOF' -[program.retries] -command = "exit 1" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 2 -exitcodes = [0] -EOF -start_server -sleep 15 -R=$(send_cmd "status retries") -echo " exit 1, startretries=2: '$R'" -echo "$R" | grep -q "FATAL" && pass "SUP-2: retries exhausted → FATAL" || fail "SUP-2: got '$R'" -stop_server - -echo "" -echo "=== SECTION H: Hot-Reload ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/reload_config.toml" -EOF -cat > /tmp/test_taskmaster/reload_config.toml << 'EOF' -[program.reloadtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -EOF -start_server -sleep 2 -PID1=$(pgrep -x sleep | head -1) -echo " PID before reload: $PID1" -R=$(send_cmd "reload") -echo "$R" | tr -d '\r' | grep -q "" && pass "HR-1: reload command accepted" || pass "HR-1: reload accepted" -kill -HUP $(pgrep -x taskmasterd) 2>/dev/null -sleep 2 -R=$(send_cmd "status reloadtest") -echo "$R" | grep -q "RUNNING\|STOPPED" && pass "HR-2: SIGHUP reload — server still responds" || fail "HR-2: got '$R'" -PID2=$(pgrep -x sleep | head -1) -echo " PID after reload: $PID2" -if [ "$PID1" = "$PID2" ] && [ -n "$PID1" ]; then - pass "HR-3: unchanged process NOT restarted (PID preserved)" -else - fail "HR-3: PID changed from $PID1 to $PID2" -fi -stop_server - -echo "" -echo "========================================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "========================================" diff --git a/tests/kill_config.toml b/tests/kill_config.toml deleted file mode 100644 index a9ec573..0000000 --- a/tests/kill_config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[program.killtest] -command = "sleep 3600" -autostart = true -autorestart = "true" -startsecs = 0 -startretries = 3 diff --git a/tests/race_config.toml b/tests/race_config.toml deleted file mode 100644 index ccae9e0..0000000 --- a/tests/race_config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[program.racetest] -command = "sleep 3600" -autostart = false -autorestart = "false" -startsecs = 0 -stopwaitsecs = 2 diff --git a/tests/race_only.sh b/tests/race_only.sh deleted file mode 100644 index ba0904e..0000000 --- a/tests/race_only.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -x sleep) 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 - if ! pgrep -x taskmasterd > /dev/null 2>&1; then - echo "FATAL: Server failed to start" - cat /tmp/test_taskmaster/server.log - exit 1 - fi -} - -stop_server() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - sleep 1 -} - -echo "=== SECTION I: Start/Stop Race Condition ===" - -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/race_config.toml" -EOF - -cat > /tmp/test_taskmaster/race_config.toml << 'EOF' -[program.racetest] -command = "sleep 3600" -autostart = false -autorestart = "false" -startsecs = 0 -stopwaitsecs = 2 -EOF - -start_server -sleep 1 - -send_cmd "start racetest" > /dev/null & -sleep 0.1 -send_cmd "stop racetest" > /dev/null -sleep 3 - -R=$(send_cmd "status racetest") -echo " After rapid start+stop: '$R'" -echo "$R" | grep -q "STOPPED" && pass "RACE-1: start then immediate stop → STOPPED" || fail "RACE-1: got '$R'" - -SLEEP_COUNT=$(pgrep -x sleep 2>/dev/null | wc -l) -[ "$SLEEP_COUNT" -eq 0 ] && pass "RACE-2: no leftover sleep processes" || fail "RACE-2: $SLEEP_COUNT sleep processes still running" - -stop_server - -echo "" -echo "========================================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -echo "========================================" diff --git a/tests/reload_config.toml b/tests/reload_config.toml deleted file mode 100644 index ecac500..0000000 --- a/tests/reload_config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[program.reloadtest] -command = "sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 diff --git a/tests/retry_config.toml b/tests/retry_config.toml deleted file mode 100644 index 01122c2..0000000 --- a/tests/retry_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.retries] -command = "exit 1" -autostart = true -autorestart = "false" -startsecs = 1 -startretries = 2 -exitcodes = [0] diff --git a/tests/run_tests.sh b/tests/run_tests.sh deleted file mode 100755 index e3804e2..0000000 --- a/tests/run_tests.sh +++ /dev/null @@ -1,398 +0,0 @@ -#!/bin/bash -# Comprehensive test script for taskmaster - -SERVER=/tmp/taskmasterd -CLIENT=/tmp/taskmasterctl -SOCK=/tmp/test_taskmaster.sock -LOGDIR=/tmp/test_taskmaster - -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -PASS=0 -FAIL=0 -BUGS="" - -pass() { echo -e "${GREEN}PASS${NC}: $1"; ((PASS++)); } -fail() { echo -e "${RED}FAIL${NC}: $1"; ((FAIL++)); BUGS="${BUGS}\n FAIL: $1"; } -bug() { echo -e "${YELLOW}BUG${NC}: $1"; ((FAIL++)); BUGS="${BUGS}\n BUG: $1"; } - -cleanup() { - kill $SERVER_PID 2>/dev/null - wait $SERVER_PID 2>/dev/null - rm -f $SOCK /tmp/test_taskmaster/server.pid - rm -f $LOGDIR/*.log - # kill any leftover sleep 3600 - pkill -f "sleep 3600" 2>/dev/null -} - -send_cmd() { - echo -ne "$1\r" | nc -U $SOCK -w 3 -} - -trap cleanup EXIT - -# Build -cd /home/samy/taskmaster -go build -o $SERVER cmd/server/main.go 2>/dev/null -go build -o $CLIENT cmd/client/main.go 2>/dev/null - -cleanup - -echo "=== Starting server ===" -cd $LOGDIR -$SERVER > /tmp/test_taskmaster/server_output.log 2>&1 & -SERVER_PID=$! -echo $SERVER_PID > /tmp/test_taskmaster/server.pid -sleep 2 - -# Check server is running -if ! kill -0 $SERVER_PID 2>/dev/null; then - echo "FATAL: Server failed to start" - cat /tmp/test_taskmaster/server_output.log - exit 1 -fi - -echo "Server started with PID $SERVER_PID" - -echo "" -echo "=== TEST 1: Basic control shell (start/stop/restart/status) ===" - -RESULT=$(send_cmd "status stayalive") -echo " status stayalive: '$RESULT'" -if echo "$RESULT" | grep -q "STOPPED"; then - pass "status shows STOPPED for non-started program" -else - fail "status should show STOPPED, got: $RESULT" -fi - -RESULT=$(send_cmd "start stayalive") -echo " start stayalive: '$RESULT'" -sleep 3 - -RESULT=$(send_cmd "status stayalive") -echo " status after start: '$RESULT'" -if echo "$RESULT" | grep -q "RUNNING"; then - pass "start works - status shows RUNNING" -else - fail "start failed - expected RUNNING, got: $RESULT" -fi - -RESULT=$(send_cmd "stop stayalive") -echo " stop stayalive: '$RESULT'" -sleep 3 - -RESULT=$(send_cmd "status stayalive") -echo " status after stop: '$RESULT'" -if echo "$RESULT" | grep -q "STOPPED"; then - pass "stop works - status shows STOPPED" -else - fail "stop failed - expected STOPPED, got: $RESULT" -fi - -send_cmd "start stayalive" > /dev/null -sleep 3 -RESULT=$(send_cmd "restart stayalive") -echo " restart stayalive: '$RESULT'" -sleep 3 - -RESULT=$(send_cmd "status stayalive") -echo " status after restart: '$RESULT'" -if echo "$RESULT" | grep -q "RUNNING"; then - pass "restart works - status shows RUNNING" -else - fail "restart failed - expected RUNNING, got: $RESULT" -fi - -RESULT=$(send_cmd "status all") -echo " status all: '$RESULT'" -if echo "$RESULT" | grep -q "stayalive"; then - pass "status all shows programs" -else - fail "status all should list programs, got: $RESULT" -fi - -echo "" -echo "=== TEST 2: Configuration - autostart ===" - -RESULT=$(send_cmd "status stayalive") -if echo "$RESULT" | grep -q "RUNNING"; then - pass "autostart=true program started automatically" -else - fail "autostart=true should be running, got: $RESULT" -fi - -RESULT=$(send_cmd "status fastexit") -if echo "$RESULT" | grep -q "STOPPED"; then - pass "autostart=false program not started" -else - fail "autostart=false should be STOPPED, got: $RESULT" -fi - -echo "" -echo "=== TEST 3: Configuration - numprocs ===" - -send_cmd "start multihello" > /dev/null -sleep 2 -RESULT=$(send_cmd "status multihello") -echo " multihello status: '$RESULT'" -if echo "$RESULT" | grep -q "multihello_0" && echo "$RESULT" | grep -q "multihello_1" && echo "$RESULT" | grep -q "multihello_2"; then - pass "numprocs=3 shows 3 processes" -else - fail "numprocs=3 should show 3 process statuses, got: $RESULT" -fi - -echo "" -echo "=== TEST 4: Configuration - autorestart ===" - -send_cmd "start fastexit" > /dev/null -sleep 3 -RESULT=$(send_cmd "status fastexit") -echo " fastexit (autorestart=false) status: '$RESULT'" -if echo "$RESULT" | grep -q "EXITED"; then - pass "autorestart=false: process exited and not restarted" -else - fail "autorestart=false: expected EXITED, got: $RESULT" -fi - -send_cmd "start error_exit" > /dev/null -sleep 3 -RESULT=$(send_cmd "status error_exit") -echo " error_exit (autorestart=true) status: '$RESULT'" -if echo "$RESULT" | grep -q "RUNNING\|BACKOFF"; then - pass "autorestart=true: process restarted after exit" -else - fail "autorestart=true: expected RUNNING/BACKOFF, got: $RESULT" -fi - -echo "" -echo "=== TEST 5: Kill supervised process -> auto restart ===" - -send_cmd "stop stayalive" > /dev/null -sleep 2 -send_cmd "start stayalive" > /dev/null -sleep 3 - -PROC_PID=$(pgrep -f "sleep 3600" | head -1) -echo " supervised process PID: $PROC_PID" -if [ -n "$PROC_PID" ]; then - kill -9 $PROC_PID 2>/dev/null - sleep 4 - RESULT=$(send_cmd "status stayalive") - echo " after kill status: '$RESULT'" - if echo "$RESULT" | grep -q "RUNNING"; then - pass "killed process was automatically restarted" - else - fail "killed process should be restarted, got: $RESULT" - fi -else - fail "could not find supervised process PID" -fi - -echo "" -echo "=== TEST 6: startretries -> abort after max retries ===" - -send_cmd "stop error_exit" > /dev/null 2>/dev/null -sleep 1 -send_cmd "start error_exit" > /dev/null 2>/dev/null -sleep 20 - -RESULT=$(send_cmd "status error_exit") -echo " after retries status: '$RESULT'" -if echo "$RESULT" | grep -q "FATAL"; then - pass "startretries exhausted -> FATAL state" -else - fail "expected FATAL after retries exhausted, got: $RESULT" -fi - -echo "" -echo "=== TEST 7: Hot-reload (SIGHUP + reload command) ===" - -RESULT=$(send_cmd "reload") -echo " reload command result: '$RESULT'" -if [ -z "$RESULT" ]; then - pass "reload command accepted (empty = success)" -else - echo "$RESULT" | grep -qi "error" && fail "reload command failed: $RESULT" || pass "reload command accepted" -fi - -kill -HUP $SERVER_PID 2>/dev/null -sleep 2 -RESULT=$(send_cmd "status stayalive") -if echo "$RESULT" | grep -q "RUNNING\|STOPPED\|EXITED"; then - pass "SIGHUP reload - server still responding" -else - fail "server not responding after SIGHUP, got: '$RESULT'" -fi - -echo "" -echo "=== TEST 8: Hot-reload - unchanged processes not restarted ===" - -send_cmd "stop stayalive" > /dev/null -sleep 2 -send_cmd "start stayalive" > /dev/null -sleep 3 - -PROC_PID1=$(pgrep -f "sleep 3600" | head -1) -echo " PID before reload: $PROC_PID1" - -kill -HUP $SERVER_PID 2>/dev/null -sleep 2 - -PROC_PID2=$(pgrep -f "sleep 3600" | head -1) -echo " PID after reload: $PROC_PID2" - -if [ "$PROC_PID1" = "$PROC_PID2" ] && [ -n "$PROC_PID1" ]; then - pass "unchanged process NOT restarted on reload (PID preserved)" -elif [ -n "$PROC_PID1" ] && [ -n "$PROC_PID2" ]; then - bug "unchanged process WAS restarted on reload (PID changed from $PROC_PID1 to $PROC_PID2)" -else - fail "could not determine PIDs (before=$PROC_PID1, after=$PROC_PID2)" -fi - -echo "" -echo "=== TEST 9: stopwaitsecs (graceful stop) ===" - -send_cmd "start graceful" > /dev/null -sleep 2 - -RESULT=$(send_cmd "status graceful") -echo " graceful before stop: '$RESULT'" - -START_TIME=$(date +%s) -send_cmd "stop graceful" > /dev/null -sleep 1 -END_TIME=$(date +%s) -ELAPSED=$((END_TIME - START_TIME)) -echo " stop command returned after ${ELAPSED}s" - -sleep 3 -RESULT=$(send_cmd "status graceful") -echo " graceful after stop: '$RESULT'" -if echo "$RESULT" | grep -q "STOPPED"; then - pass "graceful stop completed" -else - fail "graceful stop did not complete, got: $RESULT" -fi - -echo "" -echo "=== TEST 10: stdout/stderr log files ===" - -send_cmd "stop stayalive" > /dev/null -sleep 1 -send_cmd "start stayalive" > /dev/null -sleep 2 - -if [ -f "$LOGDIR/stayalive.log" ]; then - pass "stdout log file created" - echo " content: $(head -c 200 $LOGDIR/stayalive.log)" -else - fail "stdout log file not created" -fi - -if [ -f "$LOGDIR/stayalive_err.log" ]; then - pass "stderr log file created" -else - fail "stderr log file not created" -fi - -echo "" -echo "=== TEST 11: Environment variables ===" - -send_cmd "start envcheck" > /dev/null -sleep 2 - -if [ -f "$LOGDIR/env.log" ]; then - if grep -q "MY_VAR=hello42" "$LOGDIR/env.log"; then - pass "environment variable MY_VAR set correctly" - else - fail "MY_VAR=hello42 not found in env output" - fi - if grep -q "ANOTHER_VAR=world" "$LOGDIR/env.log"; then - pass "environment variable ANOTHER_VAR set correctly" - else - fail "ANOTHER_VAR=world not found in env output" - fi -else - fail "env.log not created" -fi - -echo "" -echo "=== TEST 12: Working directory ===" - -send_cmd "start dircheck" > /dev/null -sleep 2 - -if [ -f "$LOGDIR/dir.log" ]; then - CONTENT=$(cat "$LOGDIR/dir.log" | tr -d '\n') - if echo "$CONTENT" | grep -q "/tmp"; then - pass "working directory set to /tmp" - else - fail "working directory should be /tmp, got: $CONTENT" - fi -else - fail "dir.log not created" -fi - -echo "" -echo "=== TEST 13: Umask ===" - -send_cmd "start umaskcheck" > /dev/null -sleep 2 - -if [ -f "$LOGDIR/umask.log" ]; then - CONTENT=$(cat "$LOGDIR/umask.log" | tr -d '\n') - if echo "$CONTENT" | grep -q "0077\|00077"; then - pass "umask set to 0077" - else - fail "umask should be 0077, got: '$CONTENT'" - fi -else - fail "umask.log not created" -fi - -echo "" -echo "=== TEST 14: Invalid commands / non-existent jobs ===" - -RESULT=$(send_cmd "bogus") -echo " bogus command: '$RESULT'" -if echo "$RESULT" | grep -qi "unknown\|error"; then - pass "invalid command returns error" -else - fail "invalid command should return error, got: $RESULT" -fi - -RESULT=$(send_cmd "start nonexistent") -echo " start nonexistent: '$RESULT'" -if echo "$RESULT" | grep -qi "not recognized\|error\|unknown"; then - pass "non-existent job returns error" -else - fail "non-existent job should return error, got: $RESULT" -fi - -echo "" -echo "=== TEST 15: start/stop all ===" - -send_cmd "stop all" > /dev/null -sleep 3 -RESULT=$(send_cmd "status all") -echo " status after stop all: '$RESULT'" -echo "$RESULT" | grep -q "RUNNING" && fail "stop all did not stop everything" || pass "stop all works" - -send_cmd "start all" > /dev/null -sleep 3 -RESULT=$(send_cmd "status all") -echo " status after start all: '$RESULT'" -echo "$RESULT" | grep -q "RUNNING" && pass "start all works" || fail "start all did not start autostart programs" - -echo "" -echo "==============================" -echo -e "RESULTS: ${GREEN}$PASS passed${NC}, ${RED}$FAIL failed${NC}" -if [ $FAIL -gt 0 ]; then - echo "" - echo "Failures/Bugs:" - echo -e "$BUGS" -fi -echo "==============================" diff --git a/tests/secs_config.toml b/tests/secs_config.toml deleted file mode 100644 index 43c3342..0000000 --- a/tests/secs_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.earlyexit] -command = "exit 0" -autostart = true -autorestart = "false" -startsecs = 5 -startretries = 1 -exitcodes = [0] diff --git a/tests/section_a.sh b/tests/section_a.sh deleted file mode 100644 index 634e6ca..0000000 --- a/tests/section_a.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' -PASS=0 -FAIL=0 - -pass() { echo -e " ${GREEN}PASS${NC}: $1"; PASS=$((PASS+1)); } -fail() { echo -e " ${RED}FAIL${NC}: $1"; FAIL=$((FAIL+1)); } - -send_cmd() { - echo -ne "$1\r" | nc -U /tmp/test_taskmaster.sock -w 3 2>/dev/null -} - -cleanup() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - kill -9 $(pgrep -x sleep) 2>/dev/null || true - rm -f /tmp/test_taskmaster.sock - rm -f /tmp/test_taskmaster/*.log - sleep 1 -} - -start_server() { - cleanup - cd /tmp/test_taskmaster - /tmp/taskmasterd > /tmp/test_taskmaster/server.log 2>&1 & - sleep 2 -} - -stop_server() { - kill -9 $(pgrep -x taskmasterd) 2>/dev/null || true - sleep 1 -} - -echo "=== SECTION A: Control Shell ===" -cat > /tmp/test_taskmaster/setup.toml << 'EOF' -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/shell_config.toml" -EOF -cat > /tmp/test_taskmaster/shell_config.toml << 'EOF' -[program.procname] -command = "sleep 3600" -autostart = false -autorestart = "false" -EOF - -start_server - -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-1: status STOPPED" || fail "CS-1: got '$R'" - -R=$(send_cmd "start procname"); sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2a: start → RUNNING" || fail "CS-2a: got '$R'" - -R=$(send_cmd "stop procname"); sleep 2 -R=$(send_cmd "status procname") -echo "$R" | grep -q "STOPPED" && pass "CS-2b: stop → STOPPED" || fail "CS-2b: got '$R'" - -send_cmd "start procname" > /dev/null; sleep 2 -send_cmd "restart procname" > /dev/null; sleep 3 -R=$(send_cmd "status procname") -echo "$R" | grep -q "RUNNING" && pass "CS-2c: restart → RUNNING" || fail "CS-2c: got '$R'" - -R=$(send_cmd "status all") -echo "$R" | grep -q "procname" && pass "CS-3: status all" || fail "CS-3: got '$R'" - -R=$(send_cmd "bogus") -echo "$R" | grep -qi "unknown\|error" && pass "CS-6: invalid cmd error" || fail "CS-6: got '$R'" - -R=$(send_cmd "start nonexistent") -echo "$R" | grep -qi "not recognized\|error" && pass "CS-7: nonexistent error" || fail "CS-7: got '$R'" - -R=$(send_cmd "reload") -echo "$R" | tr -d '\r' | grep -q "" && pass "CS-5: reload accepted" || pass "CS-5: reload accepted (empty)" - -stop_server -echo "A: $PASS passed, $FAIL failed" diff --git a/tests/setup.toml b/tests/setup.toml deleted file mode 100644 index cf65bdb..0000000 --- a/tests/setup.toml +++ /dev/null @@ -1,3 +0,0 @@ -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/attach_config.toml" diff --git a/tests/setup_ar.toml b/tests/setup_ar.toml deleted file mode 100644 index 74df9c5..0000000 --- a/tests/setup_ar.toml +++ /dev/null @@ -1,3 +0,0 @@ -socket = "/tmp/test_taskmaster.sock" -prompt = "taskmaster> " -config = "/tmp/test_taskmaster/ar_config.toml" diff --git a/tests/shell_config.toml b/tests/shell_config.toml deleted file mode 100644 index f29656b..0000000 --- a/tests/shell_config.toml +++ /dev/null @@ -1,4 +0,0 @@ -[program.procname] -command = "sleep 3600" -autostart = false -autorestart = "false" diff --git a/tests/stop_config.toml b/tests/stop_config.toml deleted file mode 100644 index e3f1742..0000000 --- a/tests/stop_config.toml +++ /dev/null @@ -1,7 +0,0 @@ -[program.termtest] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 3 diff --git a/tests/stopwait_config.toml b/tests/stopwait_config.toml deleted file mode 100644 index 90e2f6a..0000000 --- a/tests/stopwait_config.toml +++ /dev/null @@ -1,8 +0,0 @@ -[program.slowstop] -command = "trap '' TERM; sleep 3600" -autostart = true -autorestart = "false" -numprocs = 1 -startsecs = 0 -stopsignal = "TERM" -stopwaitsecs = 2 diff --git a/tests/unexpected_config.toml b/tests/unexpected_config.toml deleted file mode 100644 index 82f8dcf..0000000 --- a/tests/unexpected_config.toml +++ /dev/null @@ -1,8 +0,0 @@ -[program.exit0] -command = "exit 0" -autostart = true -autorestart = "unexpected" -numprocs = 1 -startsecs = 0 -startretries = 3 -exitcodes = [0] diff --git a/tests/unexpected_config2.toml b/tests/unexpected_config2.toml deleted file mode 100644 index c3ca34b..0000000 --- a/tests/unexpected_config2.toml +++ /dev/null @@ -1,8 +0,0 @@ -[program.exitbad] -command = "exit 42" -autostart = true -autorestart = "unexpected" -numprocs = 1 -startsecs = 0 -startretries = 3 -exitcodes = [0] From 012ccedc3edcef5de95d0565950c7a39b0dce2a8 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Mon, 24 Aug 2026 13:31:31 +0100 Subject: [PATCH 6/9] refactor: use INFO instead of DEBUG --- compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose.yaml b/compose.yaml index 2faff6e..9df11d9 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,7 +8,7 @@ services: ports: - "8080:8080" environment: - LOG_LEVEL: DEBUG + LOG_LEVEL: INFO volumes: - /dev/log:/dev/log - ./taskmaster.toml:/app/taskmaster.toml \ No newline at end of file From 936491f43da1ab5ed17e239af256931fb80d060d Mon Sep 17 00:00:00 2001 From: mylastresort Date: Tue, 25 Aug 2026 12:47:20 +0100 Subject: [PATCH 7/9] fix: revert back umask command impl --- internal/job/job.go | 12 +----------- internal/job/start.go | 12 ++---------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/internal/job/job.go b/internal/job/job.go index 64aa082..8421c88 100644 --- a/internal/job/job.go +++ b/internal/job/job.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "os/exec" - "strconv" "sync" "syscall" "time" @@ -13,10 +12,6 @@ import ( "github.com/Archer-01/taskmaster/internal/utils" ) -// serializes umask switching around process starts, since umask is -// process-wide and concurrent starts could inherit the wrong one -var startMu sync.Mutex - func (p *Job) StartCmd(procId int) error { if err := p.cmds[procId].Start(); err != nil { return err @@ -151,9 +146,4 @@ func (j *Job) setLog(file string, writer *utils.DynamicWriter, _default io.Write return nil } -func parseUmask(s string) int { - if v, err := strconv.ParseUint(s, 8, 32); err == nil { - return int(v) - } - return 0o022 -} + diff --git a/internal/job/start.go b/internal/job/start.go index 005a18d..d90d7ee 100644 --- a/internal/job/start.go +++ b/internal/job/start.go @@ -1,6 +1,7 @@ package job import ( + "fmt" "os" "os/exec" "sync" @@ -8,7 +9,6 @@ import ( "time" "github.com/Archer-01/taskmaster/internal/logger" - "github.com/Archer-01/taskmaster/internal/parser/config" ) func (j *Job) closeStartReady(procId int) { @@ -88,11 +88,7 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { usePgid = 0 } - argv := config.ParseCommand(j.Command) - if len(argv) == 0 { - argv = []string{j.Command} - } - cmd := exec.Command(argv[0], argv[1:]...) + cmd := exec.Command("sh", "-c", fmt.Sprintf("umask %v && %v", j.Umask, j.Command)) cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pgid: usePgid} j.cmds[id] = cmd @@ -213,11 +209,7 @@ func (j *Job) tryStart(procId int) error { j.cmds[procId].Env = append(j.Environment, os.Environ()...) j.cmds[procId].Dir = j.Dir - startMu.Lock() - old := syscall.Umask(parseUmask(j.Umask)) err = j.StartCmd(procId) - syscall.Umask(old) - startMu.Unlock() if err != nil { return err } From dd6f938c55b8a82761f6f960b3e4afcca64fdf59 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Wed, 26 Aug 2026 22:19:28 +0100 Subject: [PATCH 8/9] fix: enhance process management and cleanup during stop --- internal/job/states.go | 4 ++++ internal/job/stop.go | 4 ++++ taskmaster.toml | 47 +++++++++++++++++++++--------------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/internal/job/states.go b/internal/job/states.go index 8865393..8345a51 100644 --- a/internal/job/states.go +++ b/internal/job/states.go @@ -57,6 +57,10 @@ func (j *Job) IsRunning() bool { return false } +func (j *Job) isRunningProc(procId int) bool { + return j._running[procId] +} + func (j *Job) procAlive(id int) bool { return j._running[id] && j.HasPgid(id) && groupAlive(j.pgid[id]) } diff --git a/internal/job/stop.go b/internal/job/stop.go index b93dd76..c355bf8 100644 --- a/internal/job/stop.go +++ b/internal/job/stop.go @@ -110,6 +110,10 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr for i := range j.NumProcs { logger.Debugf("Stop(): Waiting for process %s to exit", j.DisplayName(i)) j.cmds[i].Wait() + for j._running[i] { + time.Sleep(100 * time.Millisecond) + logger.Debugf("Stop(): Waiting for process %s to finish cleanup", j.DisplayName(i)) + } } } return nil diff --git a/taskmaster.toml b/taskmaster.toml index 461efee..0c66701 100644 --- a/taskmaster.toml +++ b/taskmaster.toml @@ -6,33 +6,34 @@ stopsignal = "INT" stdout_logfile = "one.log" autostart = true autorestart = "true" -# startsecs = 100 -numprocs = 10 +startsecs = 100 +numprocs = 300 +umask = "0012" startretries = 3 # user = "taskmaster" -[program.one] -command = "ping localhost" -stopsignal = "TERM" -stdout_logfile = "one.log" -# autostart = false +# [program.one] +# command = "ping localhost" +# stopsignal = "TERM" +# stdout_logfile = "one.log" +# # autostart = false -[program.two] -command = "ping 127.0.0.1" -stopsignal = "INT" -stdout_logfile = "two.log" -# autostart = false -startsecs = 5 -startretries = 10 +# [program.two] +# command = "ping 127.0.0.1" +# stopsignal = "INT" +# stdout_logfile = "two.log" +# # autostart = false +# startsecs = 5 +# startretries = 10 -[program.three] -# autostart = false -command = "ping 0.0.0.0" -stdout_logfile = "three.log" +# [program.three] +# # autostart = false +# command = "ping 0.0.0.0" +# stdout_logfile = "three.log" -[program.four] -# autostart = false -command = "ping google.com" -stdout_logfile = "four.log" -numprocs = 3 \ No newline at end of file +# [program.four] +# # autostart = false +# command = "ping google.com" +# stdout_logfile = "four.log" +# numprocs = 3 \ No newline at end of file From c53a52999551d0652132873a6ae723c414ce30c8 Mon Sep 17 00:00:00 2001 From: mylastresort Date: Thu, 27 Aug 2026 20:11:12 +0100 Subject: [PATCH 9/9] fix: prevent crash and deadlock when shrinking numprocs on reload Reloading with a smaller numprocs previously raced the shrink-Resize against still-running worker goroutines, causing an index-out-of-range panic (e.g. "[64] with length 3"), and trying to wait for these workers exposed a double-Wait deadlock and a restart-on-stop race. Stop() no longer calls cmds[i].Wait() (the worker already waits on the same *exec.Cmd, so a second concurrent Wait never returns). _running[i] is now owned solely by the worker's deferred cleanup, which runs last, so Stop waits for _running[i] to clear - re-killing the worker's current process group (which can change on restart) until it terminates - before returning. This guarantees all removed workers have fully exited before Resize truncates the per-proc slices. --- internal/job/start.go | 4 +++- internal/job/stop.go | 45 +++++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/internal/job/start.go b/internal/job/start.go index d90d7ee..f113704 100644 --- a/internal/job/start.go +++ b/internal/job/start.go @@ -71,6 +71,9 @@ func (j *Job) Start(wg *sync.WaitGroup, _done chan bool, procId int, count int) func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { wg.Add(1) + defer func() { + j._running[id] = false + }() defer wg.Done() defer j.closeStartReady(id) @@ -185,7 +188,6 @@ func (j *Job) startJobWorker(wg *sync.WaitGroup, id int, pgid int) { } else if j.Is(STOPPING, id) { j.SetState(STOPPED, id) } - j._running[id] = false } func (j *Job) tryStart(procId int) error { diff --git a/internal/job/stop.go b/internal/job/stop.go index c355bf8..7703247 100644 --- a/internal/job/stop.go +++ b/internal/job/stop.go @@ -46,14 +46,18 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr st := func(i int) { defer _wg.Done() logger.Debugf("Stop(): Stopping process %s", j.DisplayName(i)) - // j.muproc.Lock() - if !j.procAlive(i) { - // j.muproc.Unlock() - } else { - logger.Debugf("Stop(): Sending stop signal to process %s", j.DisplayName(i)) - j.SetState(STOPPING, i) - // j.muproc.Unlock() + if !j._running[i] { + // Worker inactive, nothing left to signal or wait for. + j.SetPgid(i, 0) + j.SetState(STOPPED, i) + return + } + + // Keep STOPPING set (don't move to STOPPED here) so the worker observes it and terminates. + j.SetState(STOPPING, i) + if j.procAlive(i) { + logger.Debugf("Stop(): Sending stop signal to process %s", j.DisplayName(i)) err := syscall.Kill(-j.pgid[i], j.StopSignal) logger.Debugf("Stop(): Sent stop signal to process %s", j.DisplayName(i)) if err != nil && err != syscall.ESRCH { @@ -76,12 +80,7 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr } } } - // j.muproc.Lock() - logger.Debugf("Stop(): Process %s stopped", j.DisplayName(i)) j.SetPgid(i, 0) - j._running[i] = false - j.SetState(STOPPED, i) - // j.muproc.Unlock() } if startProcId >= 0 && startProcId < j.NumProcs { @@ -101,16 +100,18 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr if startProcId >= 0 && startProcId < j.NumProcs { for i := startProcId; i < startProcId+countProcId && i < j.NumProcs; i++ { - logger.Debugf("Stop(): Waiting for process %s to exit", j.DisplayName(i)) - j.cmds[i].Wait() - logger.Debugf("Stop(): Process %s exited", j.DisplayName(i)) + // Don't call cmds[i].Wait() (double Wait deadlocks); wait for the worker and re-kill its current group until it exits. + for j._running[i] { + j.sigkillProc(i) + time.Sleep(100 * time.Millisecond) + logger.Debugf("Stop(): Waiting for process %s to finish cleanup", j.DisplayName(i)) + } } } else { logger.Debugf("Stop(): Waiting for all processes of job %s to exit", j.Name) for i := range j.NumProcs { - logger.Debugf("Stop(): Waiting for process %s to exit", j.DisplayName(i)) - j.cmds[i].Wait() for j._running[i] { + j.sigkillProc(i) time.Sleep(100 * time.Millisecond) logger.Debugf("Stop(): Waiting for process %s to finish cleanup", j.DisplayName(i)) } @@ -118,3 +119,13 @@ func (j *Job) Stop(wg *sync.WaitGroup, _done chan bool, startProcId int, countPr } return nil } + +func (j *Job) sigkillProc(i int) { + pgid := j.pgid[i] + if pgid == 0 { + return + } + if err := syscall.Kill(-pgid, syscall.SIGKILL); err != nil && err != syscall.ESRCH { + logger.Debug(err) + } +}