package cli import ( "bufio" "context" "encoding/json" "errors" "flag" "fmt" "io" "net/http" "net/url" "os" "os/exec" "os/signal" "path/filepath" "sort" "strconv" "strings" "sync" "syscall" "time" "pxmon/internal/cluster" "pxmon/internal/history" ) const ( telegramBotDefaultPoll = 2 * time.Second telegramBotRuntimeDir = "bot/telegram" telegramBotPIDFile = "bot.pid" telegramBotLogFile = "bot.log" ) func (a *App) runBot(args []string, configPath string, jsonOut bool) int { if len(args) == 0 { a.printBotHelp() return 0 } store, err := cluster.NewStore(configPath) if err != nil { fmt.Fprintf(a.err, "init config store: %v\n", err) return 1 } svc := cluster.NewService(store) svc.AttachNetworkStore(history.NewNetworkStore(svc.DataDir())) switch strings.ToLower(strings.TrimSpace(args[0])) { case "help", "-h", "--help": a.printBotHelp() return 0 case "telegram", "tg": return a.runBotTelegram(svc, configPath, args[1:], jsonOut) default: fmt.Fprintf(a.err, "unknown bot command %q\n\n", args[0]) a.printBotHelp() return 2 } } func (a *App) runBotTelegram(svc *cluster.Service, configPath string, args []string, jsonOut bool) int { if len(args) == 0 { a.printBotTelegramHelp() return 0 } switch strings.ToLower(strings.TrimSpace(args[0])) { case "help", "-h", "--help": a.printBotTelegramHelp() return 0 case "show": fs := flag.NewFlagSet("bot telegram show", flag.ContinueOnError) fs.SetOutput(a.err) showToken := fs.Bool("show-token", false, "Reveal token in plain text") if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram show [--show-token]") return 2 } cfg, err := svc.GetTelegram() if err != nil { fmt.Fprintf(a.err, "telegram show: %v\n", err) return 1 } if jsonOut { out := cfg if !*showToken { out.Token = maskSecret(out.Token) } _ = writeJSON(a.out, out) return 0 } token := printableSecret(maskSecret(cfg.Token)) if *showToken { token = printableSecret(cfg.Token) } fmt.Fprintf(a.out, "Enabled: %t\n", cfg.Enabled) fmt.Fprintf(a.out, "Token: %s\n", token) fmt.Fprintf(a.out, "Allowed IDs: %s\n", formatInt64IDs(cfg.AllowedUserIDs)) if !cfg.UpdatedAt.IsZero() { fmt.Fprintf(a.out, "Updated: %s\n", cfg.UpdatedAt.Format(time.RFC3339)) } return 0 case "set": fs := flag.NewFlagSet("bot telegram set", flag.ContinueOnError) fs.SetOutput(a.err) token := fs.String("token", "", "Telegram bot token") enable := fs.Bool("enable", true, "Enable bot after updating config") allowCSV := fs.String("allow-ids", "", "Comma-separated Telegram user IDs") var allowList int64SliceFlag fs.Var(&allowList, "allow", "Allowed Telegram user ID (repeatable)") if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram set --token --allow [--allow ...]") return 2 } ids := append([]int64(nil), allowList...) if strings.TrimSpace(*allowCSV) != "" { fromCSV, err := parseInt64CSV(*allowCSV) if err != nil { fmt.Fprintf(a.err, "telegram set: %v\n", err) return 2 } ids = append(ids, fromCSV...) } ids = normalizeInt64IDs(ids) if strings.TrimSpace(*token) == "" { fmt.Fprintln(a.err, "telegram set: --token is required") return 2 } if len(ids) == 0 { fmt.Fprintln(a.err, "telegram set: at least one allowed telegram user id is required (--allow)") return 2 } updated, err := svc.SetTelegram(cluster.Telegram{ Enabled: *enable, Token: strings.TrimSpace(*token), AllowedUserIDs: ids, }) if err != nil { fmt.Fprintf(a.err, "telegram set: %v\n", err) return 1 } if err := syncTelegramBotDaemon(svc, configPath, updated.Enabled, telegramBotDefaultPoll); err != nil { fmt.Fprintf(a.err, "telegram set: %v\n", err) return 1 } if jsonOut { out := updated out.Token = maskSecret(out.Token) _ = writeJSON(a.out, out) return 0 } fmt.Fprintln(a.out, "Telegram bot config updated.") fmt.Fprintf(a.out, "Enabled: %t | Allowed IDs: %s\n", updated.Enabled, formatInt64IDs(updated.AllowedUserIDs)) if updated.Enabled { pidPath, logPath := telegramBotPaths(svc) pid, running, _ := readTelegramBotPID(pidPath) if running && pid > 0 { fmt.Fprintf(a.out, "Background worker: running (pid %d)\n", pid) } else { fmt.Fprintln(a.out, "Background worker: starting") } fmt.Fprintf(a.out, "Logs: %s\n", logPath) } return 0 case "disable": fs := flag.NewFlagSet("bot telegram disable", flag.ContinueOnError) fs.SetOutput(a.err) if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram disable") return 2 } updated, err := svc.DisableTelegram() if err != nil { fmt.Fprintf(a.err, "telegram disable: %v\n", err) return 1 } if err := syncTelegramBotDaemon(svc, configPath, false, telegramBotDefaultPoll); err != nil { fmt.Fprintf(a.err, "telegram disable: %v\n", err) return 1 } if jsonOut { out := updated out.Token = maskSecret(out.Token) _ = writeJSON(a.out, out) return 0 } fmt.Fprintf(a.out, "Telegram bot disabled. Allowed IDs: %s\n", formatInt64IDs(updated.AllowedUserIDs)) _, logPath := telegramBotPaths(svc) fmt.Fprintln(a.out, "Background worker: stopped") fmt.Fprintf(a.out, "Logs: %s\n", logPath) return 0 case "restart": fs := flag.NewFlagSet("bot telegram restart", flag.ContinueOnError) fs.SetOutput(a.err) poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff for daemon run loop") if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram restart [--poll 2s]") return 2 } cfg, err := svc.GetTelegram() if err != nil { fmt.Fprintf(a.err, "telegram restart: %v\n", err) return 1 } if !cfg.Enabled { _ = stopTelegramBotDaemon(svc, configPath) if jsonOut { _ = writeJSON(a.out, map[string]any{ "enabled": false, "running": false, "reason": "telegram is disabled in settings", }) return 0 } fmt.Fprintln(a.out, "Telegram bot is disabled in settings; background worker remains stopped.") return 0 } pid, logPath, err := restartTelegramBotDaemon(svc, configPath, *poll) if err != nil { fmt.Fprintf(a.err, "telegram restart: %v\n", err) return 1 } if jsonOut { _ = writeJSON(a.out, map[string]any{ "enabled": true, "running": true, "pid": pid, "log_path": logPath, }) return 0 } fmt.Fprintf(a.out, "Telegram bot restarted (pid %d).\n", pid) fmt.Fprintf(a.out, "Logs: %s\n", logPath) return 0 case "logs": fs := flag.NewFlagSet("bot telegram logs", flag.ContinueOnError) fs.SetOutput(a.err) tail := fs.Int("tail", 200, "Number of last log lines") if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram logs [--tail 200]") return 2 } cfg, cfgErr := svc.GetTelegram() if cfgErr == nil && !cfg.Enabled { _ = stopTelegramBotDaemon(svc, configPath) if jsonOut { _ = writeJSON(a.out, map[string]any{ "running": false, "reason": "telegram is disabled", }) return 0 } fmt.Fprintln(a.out, "Telegram process is not running (disabled in settings).") return 0 } pidPath, logPath := telegramBotPaths(svc) pid, running, _ := readTelegramBotPID(pidPath) lines, err := readLastLines(logPath, *tail) if err != nil { fmt.Fprintf(a.err, "telegram logs: %v\n", err) return 1 } if jsonOut { _ = writeJSON(a.out, map[string]any{ "running": running, "pid": pid, "pid_path": pidPath, "log_path": logPath, "lines": lines, }) return 0 } fmt.Fprintf(a.out, "Running: %t", running) if pid > 0 { fmt.Fprintf(a.out, " (pid %d)", pid) } fmt.Fprintln(a.out) if !running { fmt.Fprintln(a.out, "Telegram process is not running.") return 0 } fmt.Fprintf(a.out, "Log file: %s\n", logPath) if len(lines) == 0 { fmt.Fprintln(a.out, "(log is empty)") return 0 } for _, line := range lines { fmt.Fprintln(a.out, line) } return 0 case "run": fs := flag.NewFlagSet("bot telegram run", flag.ContinueOnError) fs.SetOutput(a.err) poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff on errors") if err := fs.Parse(args[1:]); err != nil { if errors.Is(err, flag.ErrHelp) { return 0 } return 2 } if fs.NArg() > 0 { fmt.Fprintln(a.err, "usage: pxmon bot telegram run [--poll 2s]") return 2 } cfg, err := svc.GetTelegram() if err != nil { fmt.Fprintf(a.err, "telegram run: %v\n", err) return 1 } if strings.TrimSpace(cfg.Token) == "" { fmt.Fprintln(a.err, "telegram run: bot token is not configured (use `pxmon bot telegram set ...`)") return 1 } if len(cfg.AllowedUserIDs) == 0 { fmt.Fprintln(a.err, "telegram run: no allowed users configured") return 1 } if !cfg.Enabled { fmt.Fprintln(a.err, "telegram run: bot is disabled, enable it with `pxmon bot telegram set --enable=true ...`") return 1 } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() fmt.Fprintf(a.out, "Starting Telegram bot (allowed users: %s)\n", formatInt64IDs(cfg.AllowedUserIDs)) err = runTelegramBotLoop(ctx, svc, configPath, cfg, *poll, a.err) if err != nil && !errors.Is(err, context.Canceled) { fmt.Fprintf(a.err, "telegram run: %v\n", err) return 1 } fmt.Fprintln(a.out, "Telegram bot stopped.") return 0 default: fmt.Fprintf(a.err, "unknown telegram subcommand %q\n\n", args[0]) a.printBotTelegramHelp() return 2 } } func (a *App) printBotHelp() { fmt.Fprintln(a.out, "pxmon bot commands:") fmt.Fprintln(a.out) fmt.Fprintln(a.out, " telegram Configure and run Telegram bot integration") fmt.Fprintln(a.out) fmt.Fprintln(a.out, "Run 'pxmon bot telegram --help' for details.") } func (a *App) printBotTelegramHelp() { fmt.Fprintln(a.out, "pxmon bot telegram commands:") fmt.Fprintln(a.out) fmt.Fprintln(a.out, " show Show Telegram bot config") fmt.Fprintln(a.out, " set Set token and allowed Telegram user IDs") fmt.Fprintln(a.out, " disable Disable Telegram bot") fmt.Fprintln(a.out, " restart Restart Telegram bot background daemon") fmt.Fprintln(a.out, " logs Show bot logs") fmt.Fprintln(a.out, " run Start Telegram bot long-polling worker") fmt.Fprintln(a.out) fmt.Fprintln(a.out, "Examples:") fmt.Fprintln(a.out, " pxmon bot telegram set --token 123:ABC --allow 111111111 --allow 222222222") fmt.Fprintln(a.out, " pxmon bot telegram show") fmt.Fprintln(a.out, " pxmon bot telegram restart") fmt.Fprintln(a.out, " pxmon bot telegram logs --tail 100") fmt.Fprintln(a.out, " pxmon bot telegram run") } func syncTelegramBotDaemon(svc *cluster.Service, configPath string, enabled bool, poll time.Duration) error { if enabled { _, _, err := startTelegramBotDaemon(svc, configPath, poll) return err } return stopTelegramBotDaemon(svc, configPath) } func restartTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) { if err := stopTelegramBotDaemon(svc, configPath); err != nil { return 0, "", err } return startTelegramBotDaemon(svc, configPath, poll) } func startTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) { pidPath, logPath := telegramBotPaths(svc) if err := os.MkdirAll(filepath.Dir(pidPath), 0o700); err != nil { return 0, "", fmt.Errorf("create telegram runtime dir: %w", err) } pid, running, err := readTelegramBotPID(pidPath) if err == nil && running { return pid, logPath, nil } if err == nil && !running { _ = os.Remove(pidPath) } logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return 0, "", fmt.Errorf("open telegram log file: %w", err) } defer logFile.Close() exePath, err := os.Executable() if err != nil { return 0, "", fmt.Errorf("resolve executable path: %w", err) } cfgPath := strings.TrimSpace(configPath) if cfgPath == "" { cfgPath = svc.ConfigPath() } cmdArgs := make([]string, 0, 8) if cfgPath != "" { cmdArgs = append(cmdArgs, "--config", cfgPath) } cmdArgs = append(cmdArgs, "bot", "telegram", "run", "--poll", poll.String()) cmd := exec.Command(exePath, cmdArgs...) cmd.Stdout = logFile cmd.Stderr = logFile cmd.Stdin = nil cmd.Env = os.Environ() if err := cmd.Start(); err != nil { return 0, "", fmt.Errorf("start telegram bot daemon: %w", err) } pid = cmd.Process.Pid if pid <= 0 { return 0, "", errors.New("telegram bot daemon started with invalid pid") } _ = cmd.Process.Release() if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)+"\n"), 0o600); err != nil { return 0, "", fmt.Errorf("write telegram pid file: %w", err) } return pid, logPath, nil } func stopTelegramBotDaemon(svc *cluster.Service, configPath string) error { pidPath, _ := telegramBotPaths(svc) pid, running, err := readTelegramBotPID(pidPath) if err != nil { if errors.Is(err, os.ErrNotExist) { pid = 0 } else { return err } } cfgPath := strings.TrimSpace(configPath) if cfgPath == "" && svc != nil { cfgPath = strings.TrimSpace(svc.ConfigPath()) } targets := map[int]struct{}{} if pid > 0 { targets[pid] = struct{}{} } for _, p := range discoverTelegramBotPIDs(cfgPath) { if p > 0 { targets[p] = struct{}{} } } if len(targets) == 0 { _ = os.Remove(pidPath) return nil } if pid > 0 && !running { delete(targets, pid) } alive := make([]int, 0, len(targets)) for p := range targets { if processRunning(p) { alive = append(alive, p) } } if len(alive) == 0 { _ = os.Remove(pidPath) return nil } for _, p := range alive { _ = signalProcess(p, syscall.SIGTERM) } if waitForProcessesStop(alive, 4*time.Second) { _ = os.Remove(pidPath) return nil } for _, p := range alive { _ = signalProcess(p, syscall.SIGKILL) } if waitForProcessesStop(alive, 2*time.Second) { _ = os.Remove(pidPath) return nil } still := make([]int, 0, len(alive)) for _, p := range alive { if processRunning(p) { still = append(still, p) } } if len(still) == 0 { _ = os.Remove(pidPath) return nil } return fmt.Errorf("telegram bot process(es) still running: %v", still) } func signalProcess(pid int, sig syscall.Signal) error { if pid <= 0 { return nil } proc, err := os.FindProcess(pid) if err != nil { return nil } if err := proc.Signal(sig); err != nil { if errors.Is(err, os.ErrProcessDone) { return nil } // ESRCH: no such process if strings.Contains(strings.ToLower(err.Error()), "no such process") { return nil } return err } return nil } func waitForProcessesStop(pids []int, timeout time.Duration) bool { deadline := time.Now().Add(timeout) for { allStopped := true for _, p := range pids { if processRunning(p) { allStopped = false break } } if allStopped { return true } if time.Now().After(deadline) { return false } time.Sleep(120 * time.Millisecond) } } func discoverTelegramBotPIDs(configPath string) []int { out, err := exec.Command("ps", "-axo", "pid=,command=").Output() if err != nil { return nil } wantCfg := strings.TrimSpace(configPath) lines := strings.Split(string(out), "\n") seen := map[int]struct{}{} pids := make([]int, 0, 8) for _, line := range lines { line = strings.TrimSpace(line) if line == "" { continue } fields := strings.Fields(line) if len(fields) < 2 { continue } pid, err := strconv.Atoi(fields[0]) if err != nil || pid <= 0 { continue } cmdline := strings.Join(fields[1:], " ") if !strings.Contains(cmdline, "bot telegram run") { continue } if !strings.Contains(cmdline, "pxmon") { continue } if wantCfg != "" && !strings.Contains(cmdline, wantCfg) { continue } if _, ok := seen[pid]; ok { continue } seen[pid] = struct{}{} pids = append(pids, pid) } return pids } func telegramBotPaths(svc *cluster.Service) (string, string) { baseDir := "." if svc != nil { baseDir = svc.DataDir() } runtimeDir := filepath.Join(baseDir, telegramBotRuntimeDir) return filepath.Join(runtimeDir, telegramBotPIDFile), filepath.Join(runtimeDir, telegramBotLogFile) } func readTelegramBotPID(pidPath string) (int, bool, error) { raw, err := os.ReadFile(pidPath) if err != nil { return 0, false, err } text := strings.TrimSpace(string(raw)) if text == "" { return 0, false, fmt.Errorf("empty pid file: %s", pidPath) } pid, err := strconv.Atoi(text) if err != nil || pid <= 0 { return 0, false, fmt.Errorf("invalid pid in %s", pidPath) } return pid, processRunning(pid), nil } func processRunning(pid int) bool { if pid <= 0 { return false } out, err := exec.Command("ps", "-o", "stat=", "-p", strconv.Itoa(pid)).Output() if err != nil { return false } state := strings.TrimSpace(string(out)) if state == "" { return false } // Zombie should be treated as not running for lifecycle control. if strings.HasPrefix(state, "Z") { return false } return true } func readLastLines(path string, tail int) ([]string, error) { if tail <= 0 { tail = 200 } f, err := os.Open(path) if err != nil { if errors.Is(err, os.ErrNotExist) { return []string{}, nil } return nil, err } defer f.Close() lines := make([]string, 0, tail) buf := make([]string, tail) idx := 0 total := 0 sc := bufio.NewScanner(f) sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for sc.Scan() { line := strings.TrimRight(sc.Text(), "\r") buf[idx%tail] = line idx++ total++ } if err := sc.Err(); err != nil { return nil, err } if total == 0 { return lines, nil } start := 0 if total > tail { start = idx % tail total = tail } lines = make([]string, 0, total) for i := 0; i < total; i++ { lines = append(lines, buf[(start+i)%tail]) } return lines, nil } func runTelegramBotLoop(ctx context.Context, svc *cluster.Service, configPath string, cfg cluster.Telegram, pollBackoff time.Duration, logErr io.Writer) error { if pollBackoff < 500*time.Millisecond { pollBackoff = 500 * time.Millisecond } client := &http.Client{ Timeout: 70 * time.Second, } // Background sampler: appends a network history snapshot for every // cluster with an agent every 30s so P95/graph commands have data to // work with even when the TUI isn't running. samplerCtx, cancelSampler := context.WithCancel(ctx) defer cancelSampler() go runHistorySampler(samplerCtx, svc, 30*time.Second, logErr) var offset int64 lastVMAlertScan := time.Time{} lastNetSustainAlertScan := time.Time{} for { liveCfg, err := svc.GetTelegram() if err == nil { if !liveCfg.Enabled { fmt.Fprintln(logErr, "telegram bot disabled in config, stopping worker") return nil } if strings.TrimSpace(liveCfg.Token) == "" { fmt.Fprintln(logErr, "telegram bot token is empty in config, stopping worker") return nil } if len(liveCfg.AllowedUserIDs) == 0 { fmt.Fprintln(logErr, "telegram bot allowed_user_ids is empty in config, stopping worker") return nil } cfg = liveCfg } allowed := make(map[int64]struct{}, len(cfg.AllowedUserIDs)) for _, id := range cfg.AllowedUserIDs { allowed[id] = struct{}{} } if time.Since(lastVMAlertScan) > 60*time.Second { lastVMAlertScan = time.Now() pollAndSendVMAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr) } if time.Since(lastNetSustainAlertScan) > 60*time.Second { lastNetSustainAlertScan = time.Now() pollAndSendSustainedNetAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr) } updates, nextOffset, err := telegramGetUpdates(ctx, client, cfg.Token, offset, 50) if err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return err } fmt.Fprintf(logErr, "telegram poll failed: %v\n", err) select { case <-ctx.Done(): return ctx.Err() case <-time.After(pollBackoff): } continue } offset = nextOffset for _, upd := range updates { if upd.Message == nil { continue } msg := upd.Message text := strings.TrimSpace(msg.Text) if text == "" { continue } userID := int64(0) if msg.From != nil { userID = msg.From.ID } if _, ok := allowed[userID]; !ok { _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotAccessDeniedHTML(userID)) continue } cmd := normalizeTelegramCommand(text) if cmd == "" { continue } if cmd == "help" { _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotHelpHTML()) continue } if handled := handleBotUsageCommand(ctx, client, svc, cfg.Token, msg.Chat.ID, cmd); handled { continue } out, code := runObserverScopedCommand(svc, configPath, cmd, observerCommandOptions{ AllowShellEscape: false, StatsAutoOnce: true, BlockBotRun: true, StripANSI: true, }) if code == 0 { if sent, sendErr := trySendGraphAttachmentFromCLIOutput(ctx, client, cfg.Token, msg.Chat.ID, cmd, out); sendErr != nil { _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, fmt.Sprintf("🔴 graph\n
send failed: %s
", htmlEscapeTelegram(sendErr.Error()))) continue } else if sent { continue } } for _, chunk := range formatBotCommandReplyHTML(cmd, out, code) { _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, chunk) } } } } type tgUpdatesResponse struct { OK bool `json:"ok"` Result []tgUpdate `json:"result"` Description string `json:"description"` } type tgUpdate struct { UpdateID int64 `json:"update_id"` Message *tgMessage `json:"message"` } type tgMessage struct { MessageID int64 `json:"message_id"` From *tgUser `json:"from"` Chat tgChat `json:"chat"` Text string `json:"text"` } type tgUser struct { ID int64 `json:"id"` Username string `json:"username"` } type tgChat struct { ID int64 `json:"id"` } var botVMAlertState = struct { sync.Mutex last map[string]string lastWarnSent map[string]time.Time }{ last: map[string]string{}, lastWarnSent: map[string]time.Time{}, } var botNetSustainAlertState = struct { sync.Mutex active map[string]bool lastSent map[string]time.Time }{ active: map[string]bool{}, lastSent: map[string]time.Time{}, } func pollAndSendSustainedNetAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) { store := svc.NetworkStore() if store == nil { return } clusters, _, err := svc.List() if err != nil { return } now := time.Now().UTC() for _, c := range clusters { policy, err := svc.GetAlertPolicy(c.ID) if err != nil || !policy.NetSustainEnabled || policy.NetSustainMbps <= 0 { continue } windowMins := policy.NetSustainMinutes if windowMins <= 0 { windowMins = 60 } cooldownMins := policy.NetSustainCooldownMins if cooldownMins <= 0 { cooldownMins = 30 } window := time.Duration(windowMins) * time.Minute cooldown := time.Duration(cooldownMins) * time.Minute snaps, err := store.Load(c.ID, now.Add(-window)) if err != nil { fmt.Fprintf(logErr, "net-alert worker: %s: %v\n", c.Name, err) continue } if len(snaps) == 0 { botNetSustainAlertState.Lock() delete(botNetSustainAlertState.active, c.ID) botNetSustainAlertState.Unlock() continue } ifaces := sustainedCandidateIfaces(snaps, policy) if len(ifaces) == 0 { continue } seenKeys := make(map[string]struct{}, len(ifaces)) minCoverage := time.Duration(float64(window) * 0.9) for _, iface := range ifaces { key := c.ID + "|" + iface seenKeys[key] = struct{}{} series := history.AggregateNodeSeries(snaps, iface) if len(series) == 0 { botNetSustainAlertState.Lock() delete(botNetSustainAlertState.active, key) delete(botNetSustainAlertState.lastSent, key) botNetSustainAlertState.Unlock() continue } coverage := series[len(series)-1].Timestamp.Sub(series[0].Timestamp) if coverage < minCoverage { continue } minV := series[0].TotalMbps maxV := series[0].TotalMbps sumV := 0.0 triggered := true for _, p := range series { v := p.TotalMbps sumV += v if v < minV { minV = v } if v > maxV { maxV = v } if v < policy.NetSustainMbps { triggered = false } } botNetSustainAlertState.Lock() prevActive := botNetSustainAlertState.active[key] lastSent := botNetSustainAlertState.lastSent[key] if !triggered { botNetSustainAlertState.active[key] = false botNetSustainAlertState.Unlock() continue } if prevActive && !lastSent.IsZero() && now.Sub(lastSent) < cooldown { botNetSustainAlertState.Unlock() continue } botNetSustainAlertState.active[key] = true botNetSustainAlertState.lastSent[key] = now botNetSustainAlertState.Unlock() avgV := sumV / float64(len(series)) p95 := history.PercentileMbps(series, 95) msg := "🔴 Sustained network alert (CRITICAL)\n" + "Cluster: " + htmlEscapeTelegram(c.Name) + "\n" + "Interface: " + htmlEscapeTelegram(iface) + "\n" + fmt.Sprintf("Condition: ≥ %.1f Mbps for %d min\n", policy.NetSustainMbps, windowMins) + fmt.Sprintf("Observed: min %.1f · avg %.1f · p95 %.1f · max %.1f Mbps (%d samples)", minV, avgV, p95, maxV, len(series)) for _, id := range userIDs { _ = telegramSendHTML(ctx, client, token, id, msg) } } botNetSustainAlertState.Lock() for k := range botNetSustainAlertState.active { if !strings.HasPrefix(k, c.ID+"|") { continue } if _, ok := seenKeys[k]; ok { continue } delete(botNetSustainAlertState.active, k) delete(botNetSustainAlertState.lastSent, k) } botNetSustainAlertState.Unlock() } } func sustainedCandidateIfaces(snaps []history.NetworkSnapshot, policy cluster.AlertPolicy) []string { seen := map[string]struct{}{} out := make([]string, 0, 16) pinned := strings.TrimSpace(policy.NetSustainIface) if pinned != "" { return []string{pinned} } for _, snap := range snaps { for _, s := range snap.Interfaces { iface := strings.TrimSpace(s.Interface) if iface == "" { continue } if _, ok := seen[iface]; ok { continue } if !ifaceAllowedByFilters(iface, policy.NetSustainInclude, policy.NetSustainExclude) { continue } seen[iface] = struct{}{} out = append(out, iface) } } sort.Strings(out) return out } func ifaceAllowedByFilters(iface string, include, exclude []string) bool { n := strings.ToLower(strings.TrimSpace(iface)) if n == "" { return false } if len(include) > 0 { matched := false for _, tok := range include { t := strings.ToLower(strings.TrimSpace(tok)) if t == "" { continue } if strings.Contains(n, t) { matched = true break } } if !matched { return false } } for _, tok := range exclude { t := strings.ToLower(strings.TrimSpace(tok)) if t == "" { continue } if strings.Contains(n, t) { return false } } return true } func telegramGetUpdates(ctx context.Context, client *http.Client, token string, offset int64, timeoutSeconds int) ([]tgUpdate, int64, error) { if timeoutSeconds < 5 { timeoutSeconds = 5 } values := url.Values{} if offset > 0 { values.Set("offset", strconv.FormatInt(offset, 10)) } values.Set("timeout", strconv.Itoa(timeoutSeconds)) values.Set("allowed_updates", `["message"]`) endpoint := "https://api.telegram.org/bot" + token + "/getUpdates?" + values.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { return nil, offset, err } resp, err := client.Do(req) if err != nil { return nil, offset, err } defer resp.Body.Close() var payload tgUpdatesResponse if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { return nil, offset, err } if !payload.OK { msg := strings.TrimSpace(payload.Description) if msg == "" { msg = "telegram getUpdates returned ok=false" } return nil, offset, errors.New(msg) } next := offset for _, upd := range payload.Result { if upd.UpdateID >= next { next = upd.UpdateID + 1 } } return payload.Result, next, nil } func telegramSendText(ctx context.Context, client *http.Client, token string, chatID int64, text string) error { text = strings.TrimSpace(text) if text == "" { return nil } chunks := splitTelegramText(text, 3500) for _, chunk := range chunks { values := url.Values{} values.Set("chat_id", strconv.FormatInt(chatID, 10)) values.Set("text", chunk) endpoint := "https://api.telegram.org/bot" + token + "/sendMessage" req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode())) if err != nil { return err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := client.Do(req) if err != nil { return err } _ = resp.Body.Close() } return nil } func telegramSendHTML(ctx context.Context, client *http.Client, token string, chatID int64, html string) error { html = strings.TrimSpace(html) if html == "" { return nil } values := url.Values{} values.Set("chat_id", strconv.FormatInt(chatID, 10)) values.Set("text", html) values.Set("parse_mode", "HTML") values.Set("disable_web_page_preview", "true") endpoint := "https://api.telegram.org/bot" + token + "/sendMessage" req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode())) if err != nil { return err } req.Header.Set("Content-Type", "application/x-www-form-urlencoded") resp, err := client.Do(req) if err != nil { return err } _ = resp.Body.Close() return nil } func htmlEscapeTelegram(s string) string { s = strings.ReplaceAll(s, "&", "&") s = strings.ReplaceAll(s, "<", "<") s = strings.ReplaceAll(s, ">", ">") return s } func formatBotHelpHTML() string { return strings.Join([]string{ "✻ PXmon (Phylex Monitor) Bot", "Secure remote control for your cluster fleet.", "", "━━ Clusters", "• cluster list", "• cluster show eu-1", "• cluster ping eu-1 --agent", "• cluster stats eu-1", "", "━━ Usage & billing", "• usage eu-1 — live processes, RAM, folders, top iface", "• traffic eu-1 1h|1d|1mo|all — P95 text summary", "• graph eu-1 1d — PNG network graph with P95 line", "• p95 eu-1 eth0 30d — interface P95 + graph attachment", "", "━━ Alerts", "• alerts set eu-1 --net-mbps 300 --ram 90 --disk 90", "• alerts set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup", "• alert-vm check eu-1", "", "━━ Virtualization", "• kvm list --cluster eu-1", "• lxd top --cluster eu-1", "", "━━ Shortcuts", "/help · /start", }, "\n") } func formatBotAccessDeniedHTML(userID int64) string { idRepr := "unknown" if userID > 0 { idRepr = strconv.FormatInt(userID, 10) } return strings.Join([]string{ "🔒 Access denied", fmt.Sprintf("Your Telegram ID %s is not on the allow-list.", idRepr), "", "Ask the cluster administrator to add it via", "pxmon bot telegram allow <id>", }, "\n") } func pollAndSendVMAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) { clusters, _, err := svc.List() if err != nil { return } for _, c := range clusters { p, err := svc.GetVMAlertPolicy(c.ID) if err != nil || !p.Enabled { continue } routing, _ := svc.GetAlertRouting(c.ID) trigger, _ := svc.GetRunbookTrigger(c.ID) callCtx, cancel := context.WithTimeout(ctx, 10*time.Second) rep, err := svc.CheckVMAlerts(callCtx, c.ID) cancel() if err != nil { fmt.Fprintf(logErr, "vm-alert worker: %s: %v\n", c.Name, err) continue } state := strings.Join(rep.Warnings, "|") botVMAlertState.Lock() prev := botVMAlertState.last[c.ID] lastWarn := botVMAlertState.lastWarnSent[c.ID] if state == "" { delete(botVMAlertState.last, c.ID) delete(botVMAlertState.lastWarnSent, c.ID) } else { botVMAlertState.last[c.ID] = state } botVMAlertState.Unlock() if state == "" || state == prev { continue } isCritical := false for _, w := range rep.Warnings { if strings.Contains(strings.ToLower(w), "min_running") || strings.Contains(strings.ToLower(w), "below") { isCritical = true break } } if !isCritical { batchDur := time.Duration(routing.WarningBatchMins) * time.Minute if batchDur <= 0 { batchDur = 5 * time.Minute } if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur { continue } botVMAlertState.Lock() botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC() botVMAlertState.Unlock() } else if !routing.CriticalImmediate { // If critical-immediate is disabled, still route as warning batch. batchDur := time.Duration(routing.WarningBatchMins) * time.Minute if batchDur <= 0 { batchDur = 5 * time.Minute } if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur { continue } botVMAlertState.Lock() botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC() botVMAlertState.Unlock() } msg := "⚠️ VM alerts\n" + "Cluster: " + htmlEscapeTelegram(c.Name) + "\n" + fmt.Sprintf("Running: %d Shut off: %d\n", rep.Running, rep.ShutOff) if len(rep.ShutOffNames) > 0 { msg += "Shut off VM: " + htmlEscapeTelegram(strings.Join(rep.ShutOffNames, ", ")) + "\n" } for _, w := range rep.Warnings { msg += "• " + htmlEscapeTelegram(w) + "\n" } for _, id := range userIDs { _ = telegramSendHTML(ctx, client, token, id, msg) } // Auto-trigger runbook on VM shutoff alert. if trigger.Enabled && trigger.OnVMShutoff && rep.ShutOff > 0 && strings.TrimSpace(trigger.RunbookID) != "" { cooldown := time.Duration(trigger.CooldownMins) * time.Minute if cooldown <= 0 { cooldown = 30 * time.Minute } if trigger.LastTriggered.IsZero() || time.Since(trigger.LastTriggered) >= cooldown { if err := executeRunbookByID(svc, trigger.RunbookID); err != nil { fmt.Fprintf(logErr, "runbook trigger: %s: %v\n", c.Name, err) } else { _ = svc.TouchRunbookTrigger(c.ID, time.Now().UTC()) for _, id := range userIDs { _ = telegramSendHTML(ctx, client, token, id, "🟠 runbook auto-triggered\n"+ "Cluster: "+htmlEscapeTelegram(c.Name)+"\n"+ "Runbook: "+htmlEscapeTelegram(trigger.RunbookID)) } } } } } } func executeRunbookByID(svc *cluster.Service, runbookID string) error { rb, ok := svc.GetRunbook(runbookID) if !ok { return errors.New("runbook not found: " + runbookID) } for _, st := range rb.Steps { cmd := strings.TrimSpace(st.Command) if cmd == "" { continue } _, code := runObserverScopedCommand(svc, svc.ConfigPath(), cmd, observerCommandOptions{ AllowShellEscape: false, StatsAutoOnce: true, BlockBotRun: true, StripANSI: true, }) if code != 0 { return errors.New("step failed: " + st.Title) } } return nil } func formatBotCommandReplyHTML(cmd, out string, code int) []string { cmd = strings.TrimSpace(cmd) body := strings.TrimRight(out, "\n") var header string if code == 0 { header = fmt.Sprintf("🟢 %s", htmlEscapeTelegram(cmd)) } else { header = fmt.Sprintf("🔴 %s exit %d", htmlEscapeTelegram(cmd), code) } if strings.TrimSpace(body) == "" { if code == 0 { return []string{header + "\n
ok
"} } return []string{header + "\n
(no output)
"} } bodyChunks := splitTelegramText(htmlEscapeTelegram(body), 3600) msgs := make([]string, 0, len(bodyChunks)) for i, chunk := range bodyChunks { if i == 0 { msgs = append(msgs, header+"\n
"+chunk+"
") } else { msgs = append(msgs, "
"+chunk+"
") } } return msgs } func splitTelegramText(text string, limit int) []string { if limit < 256 { limit = 256 } if len(text) <= limit { return []string{text} } lines := strings.Split(text, "\n") out := make([]string, 0, len(lines)) var cur strings.Builder for _, line := range lines { candidate := line if cur.Len() > 0 { candidate = "\n" + line } if cur.Len()+len(candidate) > limit { if cur.Len() > 0 { out = append(out, cur.String()) cur.Reset() } for len(line) > limit { out = append(out, line[:limit]) line = line[limit:] } if line != "" { cur.WriteString(line) } continue } cur.WriteString(candidate) } if cur.Len() > 0 { out = append(out, cur.String()) } return out } func normalizeTelegramCommand(text string) string { text = strings.TrimSpace(text) if text == "" { return "" } if strings.HasPrefix(text, "/") { text = strings.TrimPrefix(text, "/") parts := strings.Fields(text) if len(parts) == 0 { return "" } parts[0] = strings.Split(parts[0], "@")[0] text = strings.Join(parts, " ") } if strings.EqualFold(text, "start") { return "help" } if strings.HasPrefix(strings.ToLower(text), "run ") { return strings.TrimSpace(text[4:]) } return text } func parseInt64CSV(raw string) ([]int64, error) { raw = strings.TrimSpace(raw) if raw == "" { return nil, nil } parts := strings.Split(raw, ",") out := make([]int64, 0, len(parts)) for _, p := range parts { p = strings.TrimSpace(p) if p == "" { continue } v, err := strconv.ParseInt(p, 10, 64) if err != nil { return nil, fmt.Errorf("invalid telegram user id %q", p) } out = append(out, v) } return out, nil } func formatInt64IDs(ids []int64) string { if len(ids) == 0 { return "(none)" } c := append([]int64(nil), ids...) sort.Slice(c, func(i, j int) bool { return c[i] < c[j] }) parts := make([]string, 0, len(c)) for _, id := range c { parts = append(parts, strconv.FormatInt(id, 10)) } return strings.Join(parts, ",") } func normalizeInt64IDs(ids []int64) []int64 { if len(ids) == 0 { return nil } seen := make(map[int64]struct{}, len(ids)) out := make([]int64, 0, len(ids)) for _, id := range ids { if id <= 0 { continue } if _, ok := seen[id]; ok { continue } seen[id] = struct{}{} out = append(out, id) } sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) if len(out) == 0 { return nil } return out } type int64SliceFlag []int64 func (s *int64SliceFlag) String() string { return formatInt64IDs(*s) } func (s *int64SliceFlag) Set(v string) error { id, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64) if err != nil { return fmt.Errorf("invalid user id %q", v) } *s = append(*s, id) return nil }