package cluster import ( "encoding/json" "errors" "fmt" "math/rand" "os" "path/filepath" "sort" "strings" "time" ) type ScheduledTask struct { ID string `json:"id"` Name string `json:"name"` Cluster string `json:"cluster,omitempty"` Command string `json:"command"` Mode string `json:"mode,omitempty"` // shell|observer Every string `json:"every"` Backoff string `json:"backoff,omitempty"` JitterSec int `json:"jitter_sec,omitempty"` RetryMax int `json:"retry_max,omitempty"` RetryCur int `json:"retry_cur,omitempty"` Enabled bool `json:"enabled"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` LastRunAt time.Time `json:"last_run_at,omitempty"` NextRunAt time.Time `json:"next_run_at,omitempty"` } type SchedulerRunResult struct { Task ScheduledTask `json:"task"` Ran bool `json:"ran"` Error string `json:"error,omitempty"` Output string `json:"output,omitempty"` ExitCode int `json:"exit_code,omitempty"` } func (s *Service) schedulerPath() string { return filepath.Join(s.DataDir(), "scheduler", "tasks.json") } func (s *Service) loadTasks() ([]ScheduledTask, error) { path := s.schedulerPath() raw, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return []ScheduledTask{}, nil } return nil, err } var items []ScheduledTask if len(strings.TrimSpace(string(raw))) == 0 { return []ScheduledTask{}, nil } if err := json.Unmarshal(raw, &items); err != nil { return nil, err } for i := range items { items[i] = normalizeScheduledTask(items[i]) } sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) }) return items, nil } func (s *Service) saveTasks(items []ScheduledTask) error { path := s.schedulerPath() if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return err } b, err := json.MarshalIndent(items, "", " ") if err != nil { return err } b = append(b, '\n') tmp := path + ".tmp" if err := os.WriteFile(tmp, b, 0o600); err != nil { return err } return os.Rename(tmp, path) } func normalizeScheduledTask(t ScheduledTask) ScheduledTask { t.ID = strings.TrimSpace(t.ID) t.Name = strings.TrimSpace(t.Name) t.Command = strings.TrimSpace(t.Command) t.Cluster = strings.TrimSpace(t.Cluster) t.Mode = strings.ToLower(strings.TrimSpace(t.Mode)) t.Every = strings.TrimSpace(t.Every) if t.ID == "" { t.ID = newClusterID() } if t.Name == "" { t.Name = t.ID } if t.Every == "" { t.Every = "5m" } if strings.TrimSpace(t.Backoff) == "" { t.Backoff = "30s" } if t.JitterSec < 0 { t.JitterSec = 0 } if t.RetryMax <= 0 { t.RetryMax = 3 } if t.RetryCur < 0 { t.RetryCur = 0 } if t.Mode == "" { t.Mode = "shell" } if t.Mode != "shell" && t.Mode != "observer" { t.Mode = "shell" } if t.CreatedAt.IsZero() { t.CreatedAt = time.Now().UTC() } if t.UpdatedAt.IsZero() { t.UpdatedAt = t.CreatedAt } return t } func parseTaskEvery(v string) (time.Duration, error) { d, err := time.ParseDuration(strings.TrimSpace(v)) if err != nil { return 0, fmt.Errorf("invalid --every duration: %w", err) } if d < time.Minute { return 0, errors.New("--every must be >= 1m") } return d, nil } func (s *Service) SchedulerList() ([]ScheduledTask, error) { return s.loadTasks() } func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) { name = strings.TrimSpace(name) command = strings.TrimSpace(command) every = strings.TrimSpace(every) mode = strings.ToLower(strings.TrimSpace(mode)) if name == "" { return ScheduledTask{}, errors.New("task name is required") } if command == "" { return ScheduledTask{}, errors.New("task command is required") } if mode == "" { mode = "shell" } if mode != "shell" && mode != "observer" { return ScheduledTask{}, errors.New("task mode must be shell|observer") } d, err := parseTaskEvery(every) if err != nil { return ScheduledTask{}, err } if strings.TrimSpace(backoff) == "" { backoff = "30s" } if _, err := time.ParseDuration(backoff); err != nil { return ScheduledTask{}, errors.New("invalid backoff duration") } if jitterSec < 0 { jitterSec = 0 } if retryMax <= 0 { retryMax = 3 } items, err := s.loadTasks() if err != nil { return ScheduledTask{}, err } for _, t := range items { if strings.EqualFold(t.Name, name) { return ScheduledTask{}, fmt.Errorf("task %q already exists", name) } } now := s.now().UTC() t := ScheduledTask{ ID: newClusterID(), Name: name, Cluster: strings.TrimSpace(clusterSel), Command: command, Mode: mode, Every: every, Backoff: backoff, JitterSec: jitterSec, RetryMax: retryMax, Enabled: enabled, CreatedAt: now, UpdatedAt: now, } if enabled { t.NextRunAt = now.Add(d) } items = append(items, t) if err := s.saveTasks(items); err != nil { return ScheduledTask{}, err } _ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every)) return t, nil } func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) { items, err := s.loadTasks() if err != nil { return ScheduledTask{}, err } selector = strings.TrimSpace(selector) if selector == "" { return ScheduledTask{}, errors.New("task name or id is required") } idx := -1 for i, t := range items { if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) { idx = i break } } if idx < 0 { return ScheduledTask{}, errors.New("task not found") } removed := items[idx] items = append(items[:idx], items[idx+1:]...) if err := s.saveTasks(items); err != nil { return ScheduledTask{}, err } _ = s.AppendChange("scheduler.remove", removed.Name, removed.Command) return removed, nil } func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error { items, err := s.loadTasks() if err != nil { return err } for i := range items { if items[i].ID != taskID { continue } items[i].LastRunAt = ranAt.UTC() if success { d, err := parseTaskEvery(items[i].Every) if err != nil { return err } items[i].RetryCur = 0 items[i].NextRunAt = items[i].LastRunAt.Add(d) } else { items[i].RetryCur++ if items[i].RetryCur > items[i].RetryMax { // cap retries and move to the next normal run window items[i].RetryCur = 0 d, err := parseTaskEvery(items[i].Every) if err != nil { return err } items[i].NextRunAt = items[i].LastRunAt.Add(d) } else { back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff)) if err != nil || back <= 0 { back = 30 * time.Second } delay := back * time.Duration(1<<(items[i].RetryCur-1)) if items[i].JitterSec > 0 { delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second } items[i].NextRunAt = items[i].LastRunAt.Add(delay) } } items[i].UpdatedAt = ranAt.UTC() break } return s.saveTasks(items) } func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) { items, err := s.loadTasks() if err != nil { return nil, err } n := now.UTC() due := make([]ScheduledTask, 0) for _, t := range items { if !t.Enabled { continue } if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) { due = append(due, t) } } sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) }) return due, nil }