chore: publish pxmon v0.2.0
This commit is contained in:
@@ -0,0 +1,301 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ScheduledTask struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
Command string `json:"command"`
|
||||
Mode string `json:"mode,omitempty"` // shell|observer
|
||||
Every string `json:"every"`
|
||||
Backoff string `json:"backoff,omitempty"`
|
||||
JitterSec int `json:"jitter_sec,omitempty"`
|
||||
RetryMax int `json:"retry_max,omitempty"`
|
||||
RetryCur int `json:"retry_cur,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastRunAt time.Time `json:"last_run_at,omitempty"`
|
||||
NextRunAt time.Time `json:"next_run_at,omitempty"`
|
||||
}
|
||||
|
||||
type SchedulerRunResult struct {
|
||||
Task ScheduledTask `json:"task"`
|
||||
Ran bool `json:"ran"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
ExitCode int `json:"exit_code,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) schedulerPath() string {
|
||||
return filepath.Join(s.DataDir(), "scheduler", "tasks.json")
|
||||
}
|
||||
|
||||
func (s *Service) loadTasks() ([]ScheduledTask, error) {
|
||||
path := s.schedulerPath()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []ScheduledTask{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var items []ScheduledTask
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return []ScheduledTask{}, nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range items {
|
||||
items[i] = normalizeScheduledTask(items[i])
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Service) saveTasks(items []ScheduledTask) error {
|
||||
path := s.schedulerPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func normalizeScheduledTask(t ScheduledTask) ScheduledTask {
|
||||
t.ID = strings.TrimSpace(t.ID)
|
||||
t.Name = strings.TrimSpace(t.Name)
|
||||
t.Command = strings.TrimSpace(t.Command)
|
||||
t.Cluster = strings.TrimSpace(t.Cluster)
|
||||
t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
|
||||
t.Every = strings.TrimSpace(t.Every)
|
||||
if t.ID == "" {
|
||||
t.ID = newClusterID()
|
||||
}
|
||||
if t.Name == "" {
|
||||
t.Name = t.ID
|
||||
}
|
||||
if t.Every == "" {
|
||||
t.Every = "5m"
|
||||
}
|
||||
if strings.TrimSpace(t.Backoff) == "" {
|
||||
t.Backoff = "30s"
|
||||
}
|
||||
if t.JitterSec < 0 {
|
||||
t.JitterSec = 0
|
||||
}
|
||||
if t.RetryMax <= 0 {
|
||||
t.RetryMax = 3
|
||||
}
|
||||
if t.RetryCur < 0 {
|
||||
t.RetryCur = 0
|
||||
}
|
||||
if t.Mode == "" {
|
||||
t.Mode = "shell"
|
||||
}
|
||||
if t.Mode != "shell" && t.Mode != "observer" {
|
||||
t.Mode = "shell"
|
||||
}
|
||||
if t.CreatedAt.IsZero() {
|
||||
t.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
if t.UpdatedAt.IsZero() {
|
||||
t.UpdatedAt = t.CreatedAt
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func parseTaskEvery(v string) (time.Duration, error) {
|
||||
d, err := time.ParseDuration(strings.TrimSpace(v))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid --every duration: %w", err)
|
||||
}
|
||||
if d < time.Minute {
|
||||
return 0, errors.New("--every must be >= 1m")
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerList() ([]ScheduledTask, error) {
|
||||
return s.loadTasks()
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
command = strings.TrimSpace(command)
|
||||
every = strings.TrimSpace(every)
|
||||
mode = strings.ToLower(strings.TrimSpace(mode))
|
||||
if name == "" {
|
||||
return ScheduledTask{}, errors.New("task name is required")
|
||||
}
|
||||
if command == "" {
|
||||
return ScheduledTask{}, errors.New("task command is required")
|
||||
}
|
||||
if mode == "" {
|
||||
mode = "shell"
|
||||
}
|
||||
if mode != "shell" && mode != "observer" {
|
||||
return ScheduledTask{}, errors.New("task mode must be shell|observer")
|
||||
}
|
||||
d, err := parseTaskEvery(every)
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
if strings.TrimSpace(backoff) == "" {
|
||||
backoff = "30s"
|
||||
}
|
||||
if _, err := time.ParseDuration(backoff); err != nil {
|
||||
return ScheduledTask{}, errors.New("invalid backoff duration")
|
||||
}
|
||||
if jitterSec < 0 {
|
||||
jitterSec = 0
|
||||
}
|
||||
if retryMax <= 0 {
|
||||
retryMax = 3
|
||||
}
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
for _, t := range items {
|
||||
if strings.EqualFold(t.Name, name) {
|
||||
return ScheduledTask{}, fmt.Errorf("task %q already exists", name)
|
||||
}
|
||||
}
|
||||
now := s.now().UTC()
|
||||
t := ScheduledTask{
|
||||
ID: newClusterID(),
|
||||
Name: name,
|
||||
Cluster: strings.TrimSpace(clusterSel),
|
||||
Command: command,
|
||||
Mode: mode,
|
||||
Every: every,
|
||||
Backoff: backoff,
|
||||
JitterSec: jitterSec,
|
||||
RetryMax: retryMax,
|
||||
Enabled: enabled,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if enabled {
|
||||
t.NextRunAt = now.Add(d)
|
||||
}
|
||||
items = append(items, t)
|
||||
if err := s.saveTasks(items); err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
_ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every))
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
selector = strings.TrimSpace(selector)
|
||||
if selector == "" {
|
||||
return ScheduledTask{}, errors.New("task name or id is required")
|
||||
}
|
||||
idx := -1
|
||||
for i, t := range items {
|
||||
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return ScheduledTask{}, errors.New("task not found")
|
||||
}
|
||||
removed := items[idx]
|
||||
items = append(items[:idx], items[idx+1:]...)
|
||||
if err := s.saveTasks(items); err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
_ = s.AppendChange("scheduler.remove", removed.Name, removed.Command)
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range items {
|
||||
if items[i].ID != taskID {
|
||||
continue
|
||||
}
|
||||
items[i].LastRunAt = ranAt.UTC()
|
||||
if success {
|
||||
d, err := parseTaskEvery(items[i].Every)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items[i].RetryCur = 0
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(d)
|
||||
} else {
|
||||
items[i].RetryCur++
|
||||
if items[i].RetryCur > items[i].RetryMax {
|
||||
// cap retries and move to the next normal run window
|
||||
items[i].RetryCur = 0
|
||||
d, err := parseTaskEvery(items[i].Every)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(d)
|
||||
} else {
|
||||
back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff))
|
||||
if err != nil || back <= 0 {
|
||||
back = 30 * time.Second
|
||||
}
|
||||
delay := back * time.Duration(1<<(items[i].RetryCur-1))
|
||||
if items[i].JitterSec > 0 {
|
||||
delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second
|
||||
}
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(delay)
|
||||
}
|
||||
}
|
||||
items[i].UpdatedAt = ranAt.UTC()
|
||||
break
|
||||
}
|
||||
return s.saveTasks(items)
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := now.UTC()
|
||||
due := make([]ScheduledTask, 0)
|
||||
for _, t := range items {
|
||||
if !t.Enabled {
|
||||
continue
|
||||
}
|
||||
if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) {
|
||||
due = append(due, t)
|
||||
}
|
||||
}
|
||||
sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) })
|
||||
return due, nil
|
||||
}
|
||||
Reference in New Issue
Block a user