pxmon/internal/cli/bot.go

package cli

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"pxmon/internal/cluster"
	"pxmon/internal/history"
)

const (
	telegramBotDefaultPoll = 2 * time.Second
	telegramBotRuntimeDir  = "bot/telegram"
	telegramBotPIDFile     = "bot.pid"
	telegramBotLogFile     = "bot.log"
)

func (a *App) runBot(args []string, configPath string, jsonOut bool) int {
	if len(args) == 0 {
		a.printBotHelp()
		return 0
	}

	store, err := cluster.NewStore(configPath)
	if err != nil {
		fmt.Fprintf(a.err, "init config store: %v\n", err)
		return 1
	}
	svc := cluster.NewService(store)
	svc.AttachNetworkStore(history.NewNetworkStore(svc.DataDir()))

	switch strings.ToLower(strings.TrimSpace(args[0])) {
	case "help", "-h", "--help":
		a.printBotHelp()
		return 0
	case "telegram", "tg":
		return a.runBotTelegram(svc, configPath, args[1:], jsonOut)
	default:
		fmt.Fprintf(a.err, "unknown bot command %q\n\n", args[0])
		a.printBotHelp()
		return 2
	}
}

func (a *App) runBotTelegram(svc *cluster.Service, configPath string, args []string, jsonOut bool) int {
	if len(args) == 0 {
		a.printBotTelegramHelp()
		return 0
	}

	switch strings.ToLower(strings.TrimSpace(args[0])) {
	case "help", "-h", "--help":
		a.printBotTelegramHelp()
		return 0
	case "show":
		fs := flag.NewFlagSet("bot telegram show", flag.ContinueOnError)
		fs.SetOutput(a.err)
		showToken := fs.Bool("show-token", false, "Reveal token in plain text")
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram show [--show-token]")
			return 2
		}

		cfg, err := svc.GetTelegram()
		if err != nil {
			fmt.Fprintf(a.err, "telegram show: %v\n", err)
			return 1
		}
		if jsonOut {
			out := cfg
			if !*showToken {
				out.Token = maskSecret(out.Token)
			}
			_ = writeJSON(a.out, out)
			return 0
		}

		token := printableSecret(maskSecret(cfg.Token))
		if *showToken {
			token = printableSecret(cfg.Token)
		}
		fmt.Fprintf(a.out, "Enabled:      %t\n", cfg.Enabled)
		fmt.Fprintf(a.out, "Token:        %s\n", token)
		fmt.Fprintf(a.out, "Allowed IDs:  %s\n", formatInt64IDs(cfg.AllowedUserIDs))
		if !cfg.UpdatedAt.IsZero() {
			fmt.Fprintf(a.out, "Updated:      %s\n", cfg.UpdatedAt.Format(time.RFC3339))
		}
		return 0

	case "set":
		fs := flag.NewFlagSet("bot telegram set", flag.ContinueOnError)
		fs.SetOutput(a.err)
		token := fs.String("token", "", "Telegram bot token")
		enable := fs.Bool("enable", true, "Enable bot after updating config")
		allowCSV := fs.String("allow-ids", "", "Comma-separated Telegram user IDs")
		var allowList int64SliceFlag
		fs.Var(&allowList, "allow", "Allowed Telegram user ID (repeatable)")
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram set --token <token> --allow <id> [--allow <id> ...]")
			return 2
		}

		ids := append([]int64(nil), allowList...)
		if strings.TrimSpace(*allowCSV) != "" {
			fromCSV, err := parseInt64CSV(*allowCSV)
			if err != nil {
				fmt.Fprintf(a.err, "telegram set: %v\n", err)
				return 2
			}
			ids = append(ids, fromCSV...)
		}
		ids = normalizeInt64IDs(ids)

		if strings.TrimSpace(*token) == "" {
			fmt.Fprintln(a.err, "telegram set: --token is required")
			return 2
		}
		if len(ids) == 0 {
			fmt.Fprintln(a.err, "telegram set: at least one allowed telegram user id is required (--allow)")
			return 2
		}

		updated, err := svc.SetTelegram(cluster.Telegram{
			Enabled:        *enable,
			Token:          strings.TrimSpace(*token),
			AllowedUserIDs: ids,
		})
		if err != nil {
			fmt.Fprintf(a.err, "telegram set: %v\n", err)
			return 1
		}
		if err := syncTelegramBotDaemon(svc, configPath, updated.Enabled, telegramBotDefaultPoll); err != nil {
			fmt.Fprintf(a.err, "telegram set: %v\n", err)
			return 1
		}

		if jsonOut {
			out := updated
			out.Token = maskSecret(out.Token)
			_ = writeJSON(a.out, out)
			return 0
		}

		fmt.Fprintln(a.out, "Telegram bot config updated.")
		fmt.Fprintf(a.out, "Enabled: %t | Allowed IDs: %s\n", updated.Enabled, formatInt64IDs(updated.AllowedUserIDs))
		if updated.Enabled {
			pidPath, logPath := telegramBotPaths(svc)
			pid, running, _ := readTelegramBotPID(pidPath)
			if running && pid > 0 {
				fmt.Fprintf(a.out, "Background worker: running (pid %d)\n", pid)
			} else {
				fmt.Fprintln(a.out, "Background worker: starting")
			}
			fmt.Fprintf(a.out, "Logs: %s\n", logPath)
		}
		return 0

	case "disable":
		fs := flag.NewFlagSet("bot telegram disable", flag.ContinueOnError)
		fs.SetOutput(a.err)
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram disable")
			return 2
		}

		updated, err := svc.DisableTelegram()
		if err != nil {
			fmt.Fprintf(a.err, "telegram disable: %v\n", err)
			return 1
		}
		if err := syncTelegramBotDaemon(svc, configPath, false, telegramBotDefaultPoll); err != nil {
			fmt.Fprintf(a.err, "telegram disable: %v\n", err)
			return 1
		}
		if jsonOut {
			out := updated
			out.Token = maskSecret(out.Token)
			_ = writeJSON(a.out, out)
			return 0
		}
		fmt.Fprintf(a.out, "Telegram bot disabled. Allowed IDs: %s\n", formatInt64IDs(updated.AllowedUserIDs))
		_, logPath := telegramBotPaths(svc)
		fmt.Fprintln(a.out, "Background worker: stopped")
		fmt.Fprintf(a.out, "Logs: %s\n", logPath)
		return 0

	case "restart":
		fs := flag.NewFlagSet("bot telegram restart", flag.ContinueOnError)
		fs.SetOutput(a.err)
		poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff for daemon run loop")
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram restart [--poll 2s]")
			return 2
		}

		cfg, err := svc.GetTelegram()
		if err != nil {
			fmt.Fprintf(a.err, "telegram restart: %v\n", err)
			return 1
		}
		if !cfg.Enabled {
			_ = stopTelegramBotDaemon(svc, configPath)
			if jsonOut {
				_ = writeJSON(a.out, map[string]any{
					"enabled": false,
					"running": false,
					"reason":  "telegram is disabled in settings",
				})
				return 0
			}
			fmt.Fprintln(a.out, "Telegram bot is disabled in settings; background worker remains stopped.")
			return 0
		}

		pid, logPath, err := restartTelegramBotDaemon(svc, configPath, *poll)
		if err != nil {
			fmt.Fprintf(a.err, "telegram restart: %v\n", err)
			return 1
		}
		if jsonOut {
			_ = writeJSON(a.out, map[string]any{
				"enabled":  true,
				"running":  true,
				"pid":      pid,
				"log_path": logPath,
			})
			return 0
		}
		fmt.Fprintf(a.out, "Telegram bot restarted (pid %d).\n", pid)
		fmt.Fprintf(a.out, "Logs: %s\n", logPath)
		return 0

	case "logs":
		fs := flag.NewFlagSet("bot telegram logs", flag.ContinueOnError)
		fs.SetOutput(a.err)
		tail := fs.Int("tail", 200, "Number of last log lines")
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram logs [--tail 200]")
			return 2
		}

		cfg, cfgErr := svc.GetTelegram()
		if cfgErr == nil && !cfg.Enabled {
			_ = stopTelegramBotDaemon(svc, configPath)
			if jsonOut {
				_ = writeJSON(a.out, map[string]any{
					"running": false,
					"reason":  "telegram is disabled",
				})
				return 0
			}
			fmt.Fprintln(a.out, "Telegram process is not running (disabled in settings).")
			return 0
		}

		pidPath, logPath := telegramBotPaths(svc)
		pid, running, _ := readTelegramBotPID(pidPath)
		lines, err := readLastLines(logPath, *tail)
		if err != nil {
			fmt.Fprintf(a.err, "telegram logs: %v\n", err)
			return 1
		}

		if jsonOut {
			_ = writeJSON(a.out, map[string]any{
				"running":  running,
				"pid":      pid,
				"pid_path": pidPath,
				"log_path": logPath,
				"lines":    lines,
			})
			return 0
		}

		fmt.Fprintf(a.out, "Running: %t", running)
		if pid > 0 {
			fmt.Fprintf(a.out, " (pid %d)", pid)
		}
		fmt.Fprintln(a.out)
		if !running {
			fmt.Fprintln(a.out, "Telegram process is not running.")
			return 0
		}
		fmt.Fprintf(a.out, "Log file: %s\n", logPath)
		if len(lines) == 0 {
			fmt.Fprintln(a.out, "(log is empty)")
			return 0
		}
		for _, line := range lines {
			fmt.Fprintln(a.out, line)
		}
		return 0

	case "run":
		fs := flag.NewFlagSet("bot telegram run", flag.ContinueOnError)
		fs.SetOutput(a.err)
		poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff on errors")
		if err := fs.Parse(args[1:]); err != nil {
			if errors.Is(err, flag.ErrHelp) {
				return 0
			}
			return 2
		}
		if fs.NArg() > 0 {
			fmt.Fprintln(a.err, "usage: pxmon bot telegram run [--poll 2s]")
			return 2
		}

		cfg, err := svc.GetTelegram()
		if err != nil {
			fmt.Fprintf(a.err, "telegram run: %v\n", err)
			return 1
		}
		if strings.TrimSpace(cfg.Token) == "" {
			fmt.Fprintln(a.err, "telegram run: bot token is not configured (use `pxmon bot telegram set ...`)")
			return 1
		}
		if len(cfg.AllowedUserIDs) == 0 {
			fmt.Fprintln(a.err, "telegram run: no allowed users configured")
			return 1
		}
		if !cfg.Enabled {
			fmt.Fprintln(a.err, "telegram run: bot is disabled, enable it with `pxmon bot telegram set --enable=true ...`")
			return 1
		}

		ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
		defer stop()

		fmt.Fprintf(a.out, "Starting Telegram bot (allowed users: %s)\n", formatInt64IDs(cfg.AllowedUserIDs))
		err = runTelegramBotLoop(ctx, svc, configPath, cfg, *poll, a.err)
		if err != nil && !errors.Is(err, context.Canceled) {
			fmt.Fprintf(a.err, "telegram run: %v\n", err)
			return 1
		}
		fmt.Fprintln(a.out, "Telegram bot stopped.")
		return 0

	default:
		fmt.Fprintf(a.err, "unknown telegram subcommand %q\n\n", args[0])
		a.printBotTelegramHelp()
		return 2
	}
}

func (a *App) printBotHelp() {
	fmt.Fprintln(a.out, "pxmon bot commands:")
	fmt.Fprintln(a.out)
	fmt.Fprintln(a.out, "  telegram   Configure and run Telegram bot integration")
	fmt.Fprintln(a.out)
	fmt.Fprintln(a.out, "Run 'pxmon bot telegram --help' for details.")
}

func (a *App) printBotTelegramHelp() {
	fmt.Fprintln(a.out, "pxmon bot telegram commands:")
	fmt.Fprintln(a.out)
	fmt.Fprintln(a.out, "  show       Show Telegram bot config")
	fmt.Fprintln(a.out, "  set        Set token and allowed Telegram user IDs")
	fmt.Fprintln(a.out, "  disable    Disable Telegram bot")
	fmt.Fprintln(a.out, "  restart    Restart Telegram bot background daemon")
	fmt.Fprintln(a.out, "  logs       Show bot logs")
	fmt.Fprintln(a.out, "  run        Start Telegram bot long-polling worker")
	fmt.Fprintln(a.out)
	fmt.Fprintln(a.out, "Examples:")
	fmt.Fprintln(a.out, "  pxmon bot telegram set --token 123:ABC --allow 111111111 --allow 222222222")
	fmt.Fprintln(a.out, "  pxmon bot telegram show")
	fmt.Fprintln(a.out, "  pxmon bot telegram restart")
	fmt.Fprintln(a.out, "  pxmon bot telegram logs --tail 100")
	fmt.Fprintln(a.out, "  pxmon bot telegram run")
}

func syncTelegramBotDaemon(svc *cluster.Service, configPath string, enabled bool, poll time.Duration) error {
	if enabled {
		_, _, err := startTelegramBotDaemon(svc, configPath, poll)
		return err
	}
	return stopTelegramBotDaemon(svc, configPath)
}

func restartTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) {
	if err := stopTelegramBotDaemon(svc, configPath); err != nil {
		return 0, "", err
	}
	return startTelegramBotDaemon(svc, configPath, poll)
}

func startTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) {
	pidPath, logPath := telegramBotPaths(svc)
	if err := os.MkdirAll(filepath.Dir(pidPath), 0o700); err != nil {
		return 0, "", fmt.Errorf("create telegram runtime dir: %w", err)
	}

	pid, running, err := readTelegramBotPID(pidPath)
	if err == nil && running {
		return pid, logPath, nil
	}
	if err == nil && !running {
		_ = os.Remove(pidPath)
	}

	logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
	if err != nil {
		return 0, "", fmt.Errorf("open telegram log file: %w", err)
	}
	defer logFile.Close()

	exePath, err := os.Executable()
	if err != nil {
		return 0, "", fmt.Errorf("resolve executable path: %w", err)
	}

	cfgPath := strings.TrimSpace(configPath)
	if cfgPath == "" {
		cfgPath = svc.ConfigPath()
	}

	cmdArgs := make([]string, 0, 8)
	if cfgPath != "" {
		cmdArgs = append(cmdArgs, "--config", cfgPath)
	}
	cmdArgs = append(cmdArgs, "bot", "telegram", "run", "--poll", poll.String())

	cmd := exec.Command(exePath, cmdArgs...)
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	cmd.Stdin = nil
	cmd.Env = os.Environ()

	if err := cmd.Start(); err != nil {
		return 0, "", fmt.Errorf("start telegram bot daemon: %w", err)
	}

	pid = cmd.Process.Pid
	if pid <= 0 {
		return 0, "", errors.New("telegram bot daemon started with invalid pid")
	}
	_ = cmd.Process.Release()
	if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)+"\n"), 0o600); err != nil {
		return 0, "", fmt.Errorf("write telegram pid file: %w", err)
	}
	return pid, logPath, nil
}

func stopTelegramBotDaemon(svc *cluster.Service, configPath string) error {
	pidPath, _ := telegramBotPaths(svc)
	pid, running, err := readTelegramBotPID(pidPath)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			pid = 0
		} else {
			return err
		}
	}

	cfgPath := strings.TrimSpace(configPath)
	if cfgPath == "" && svc != nil {
		cfgPath = strings.TrimSpace(svc.ConfigPath())
	}

	targets := map[int]struct{}{}
	if pid > 0 {
		targets[pid] = struct{}{}
	}
	for _, p := range discoverTelegramBotPIDs(cfgPath) {
		if p > 0 {
			targets[p] = struct{}{}
		}
	}
	if len(targets) == 0 {
		_ = os.Remove(pidPath)
		return nil
	}
	if pid > 0 && !running {
		delete(targets, pid)
	}

	alive := make([]int, 0, len(targets))
	for p := range targets {
		if processRunning(p) {
			alive = append(alive, p)
		}
	}
	if len(alive) == 0 {
		_ = os.Remove(pidPath)
		return nil
	}

	for _, p := range alive {
		_ = signalProcess(p, syscall.SIGTERM)
	}
	if waitForProcessesStop(alive, 4*time.Second) {
		_ = os.Remove(pidPath)
		return nil
	}

	for _, p := range alive {
		_ = signalProcess(p, syscall.SIGKILL)
	}
	if waitForProcessesStop(alive, 2*time.Second) {
		_ = os.Remove(pidPath)
		return nil
	}

	still := make([]int, 0, len(alive))
	for _, p := range alive {
		if processRunning(p) {
			still = append(still, p)
		}
	}
	if len(still) == 0 {
		_ = os.Remove(pidPath)
		return nil
	}
	return fmt.Errorf("telegram bot process(es) still running: %v", still)
}

func signalProcess(pid int, sig syscall.Signal) error {
	if pid <= 0 {
		return nil
	}
	proc, err := os.FindProcess(pid)
	if err != nil {
		return nil
	}
	if err := proc.Signal(sig); err != nil {
		if errors.Is(err, os.ErrProcessDone) {
			return nil
		}
		// ESRCH: no such process
		if strings.Contains(strings.ToLower(err.Error()), "no such process") {
			return nil
		}
		return err
	}
	return nil
}

func waitForProcessesStop(pids []int, timeout time.Duration) bool {
	deadline := time.Now().Add(timeout)
	for {
		allStopped := true
		for _, p := range pids {
			if processRunning(p) {
				allStopped = false
				break
			}
		}
		if allStopped {
			return true
		}
		if time.Now().After(deadline) {
			return false
		}
		time.Sleep(120 * time.Millisecond)
	}
}

func discoverTelegramBotPIDs(configPath string) []int {
	out, err := exec.Command("ps", "-axo", "pid=,command=").Output()
	if err != nil {
		return nil
	}

	wantCfg := strings.TrimSpace(configPath)
	lines := strings.Split(string(out), "\n")
	seen := map[int]struct{}{}
	pids := make([]int, 0, 8)
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		pid, err := strconv.Atoi(fields[0])
		if err != nil || pid <= 0 {
			continue
		}
		cmdline := strings.Join(fields[1:], " ")
		if !strings.Contains(cmdline, "bot telegram run") {
			continue
		}
		if !strings.Contains(cmdline, "pxmon") {
			continue
		}
		if wantCfg != "" && !strings.Contains(cmdline, wantCfg) {
			continue
		}
		if _, ok := seen[pid]; ok {
			continue
		}
		seen[pid] = struct{}{}
		pids = append(pids, pid)
	}
	return pids
}

func telegramBotPaths(svc *cluster.Service) (string, string) {
	baseDir := "."
	if svc != nil {
		baseDir = svc.DataDir()
	}
	runtimeDir := filepath.Join(baseDir, telegramBotRuntimeDir)
	return filepath.Join(runtimeDir, telegramBotPIDFile), filepath.Join(runtimeDir, telegramBotLogFile)
}

func readTelegramBotPID(pidPath string) (int, bool, error) {
	raw, err := os.ReadFile(pidPath)
	if err != nil {
		return 0, false, err
	}
	text := strings.TrimSpace(string(raw))
	if text == "" {
		return 0, false, fmt.Errorf("empty pid file: %s", pidPath)
	}
	pid, err := strconv.Atoi(text)
	if err != nil || pid <= 0 {
		return 0, false, fmt.Errorf("invalid pid in %s", pidPath)
	}
	return pid, processRunning(pid), nil
}

func processRunning(pid int) bool {
	if pid <= 0 {
		return false
	}
	out, err := exec.Command("ps", "-o", "stat=", "-p", strconv.Itoa(pid)).Output()
	if err != nil {
		return false
	}
	state := strings.TrimSpace(string(out))
	if state == "" {
		return false
	}
	// Zombie should be treated as not running for lifecycle control.
	if strings.HasPrefix(state, "Z") {
		return false
	}
	return true
}

func readLastLines(path string, tail int) ([]string, error) {
	if tail <= 0 {
		tail = 200
	}

	f, err := os.Open(path)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return []string{}, nil
		}
		return nil, err
	}
	defer f.Close()

	lines := make([]string, 0, tail)
	buf := make([]string, tail)
	idx := 0
	total := 0

	sc := bufio.NewScanner(f)
	sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
	for sc.Scan() {
		line := strings.TrimRight(sc.Text(), "\r")
		buf[idx%tail] = line
		idx++
		total++
	}
	if err := sc.Err(); err != nil {
		return nil, err
	}

	if total == 0 {
		return lines, nil
	}

	start := 0
	if total > tail {
		start = idx % tail
		total = tail
	}
	lines = make([]string, 0, total)
	for i := 0; i < total; i++ {
		lines = append(lines, buf[(start+i)%tail])
	}
	return lines, nil
}

func runTelegramBotLoop(ctx context.Context, svc *cluster.Service, configPath string, cfg cluster.Telegram, pollBackoff time.Duration, logErr io.Writer) error {
	if pollBackoff < 500*time.Millisecond {
		pollBackoff = 500 * time.Millisecond
	}

	client := &http.Client{
		Timeout: 70 * time.Second,
	}

	// Background sampler: appends a network history snapshot for every
	// cluster with an agent every 30s so P95/graph commands have data to
	// work with even when the TUI isn't running.
	samplerCtx, cancelSampler := context.WithCancel(ctx)
	defer cancelSampler()
	go runHistorySampler(samplerCtx, svc, 30*time.Second, logErr)

	var offset int64
	lastVMAlertScan := time.Time{}
	lastNetSustainAlertScan := time.Time{}
	for {
		liveCfg, err := svc.GetTelegram()
		if err == nil {
			if !liveCfg.Enabled {
				fmt.Fprintln(logErr, "telegram bot disabled in config, stopping worker")
				return nil
			}
			if strings.TrimSpace(liveCfg.Token) == "" {
				fmt.Fprintln(logErr, "telegram bot token is empty in config, stopping worker")
				return nil
			}
			if len(liveCfg.AllowedUserIDs) == 0 {
				fmt.Fprintln(logErr, "telegram bot allowed_user_ids is empty in config, stopping worker")
				return nil
			}
			cfg = liveCfg
		}

		allowed := make(map[int64]struct{}, len(cfg.AllowedUserIDs))
		for _, id := range cfg.AllowedUserIDs {
			allowed[id] = struct{}{}
		}
		if time.Since(lastVMAlertScan) > 60*time.Second {
			lastVMAlertScan = time.Now()
			pollAndSendVMAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr)
		}
		if time.Since(lastNetSustainAlertScan) > 60*time.Second {
			lastNetSustainAlertScan = time.Now()
			pollAndSendSustainedNetAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr)
		}

		updates, nextOffset, err := telegramGetUpdates(ctx, client, cfg.Token, offset, 50)
		if err != nil {
			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
				return err
			}
			fmt.Fprintf(logErr, "telegram poll failed: %v\n", err)
			select {
			case <-ctx.Done():
				return ctx.Err()
			case <-time.After(pollBackoff):
			}
			continue
		}
		offset = nextOffset

		for _, upd := range updates {
			if upd.Message == nil {
				continue
			}
			msg := upd.Message
			text := strings.TrimSpace(msg.Text)
			if text == "" {
				continue
			}

			userID := int64(0)
			if msg.From != nil {
				userID = msg.From.ID
			}
			if _, ok := allowed[userID]; !ok {
				_ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotAccessDeniedHTML(userID))
				continue
			}

			cmd := normalizeTelegramCommand(text)
			if cmd == "" {
				continue
			}

			if cmd == "help" {
				_ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotHelpHTML())
				continue
			}

			if handled := handleBotUsageCommand(ctx, client, svc, cfg.Token, msg.Chat.ID, cmd); handled {
				continue
			}

			out, code := runObserverScopedCommand(svc, configPath, cmd, observerCommandOptions{
				AllowShellEscape: false,
				StatsAutoOnce:    true,
				BlockBotRun:      true,
				StripANSI:        true,
			})
			if code == 0 {
				if sent, sendErr := trySendGraphAttachmentFromCLIOutput(ctx, client, cfg.Token, msg.Chat.ID, cmd, out); sendErr != nil {
					_ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID,
						fmt.Sprintf("🔴 <b>graph</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
					continue
				} else if sent {
					continue
				}
			}
			for _, chunk := range formatBotCommandReplyHTML(cmd, out, code) {
				_ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, chunk)
			}
		}
	}
}

type tgUpdatesResponse struct {
	OK          bool       `json:"ok"`
	Result      []tgUpdate `json:"result"`
	Description string     `json:"description"`
}

type tgUpdate struct {
	UpdateID int64      `json:"update_id"`
	Message  *tgMessage `json:"message"`
}

type tgMessage struct {
	MessageID int64   `json:"message_id"`
	From      *tgUser `json:"from"`
	Chat      tgChat  `json:"chat"`
	Text      string  `json:"text"`
}

type tgUser struct {
	ID       int64  `json:"id"`
	Username string `json:"username"`
}

type tgChat struct {
	ID int64 `json:"id"`
}

var botVMAlertState = struct {
	sync.Mutex
	last         map[string]string
	lastWarnSent map[string]time.Time
}{
	last:         map[string]string{},
	lastWarnSent: map[string]time.Time{},
}

var botNetSustainAlertState = struct {
	sync.Mutex
	active   map[string]bool
	lastSent map[string]time.Time
}{
	active:   map[string]bool{},
	lastSent: map[string]time.Time{},
}

func pollAndSendSustainedNetAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) {
	store := svc.NetworkStore()
	if store == nil {
		return
	}
	clusters, _, err := svc.List()
	if err != nil {
		return
	}
	now := time.Now().UTC()

	for _, c := range clusters {
		policy, err := svc.GetAlertPolicy(c.ID)
		if err != nil || !policy.NetSustainEnabled || policy.NetSustainMbps <= 0 {
			continue
		}

		windowMins := policy.NetSustainMinutes
		if windowMins <= 0 {
			windowMins = 60
		}
		cooldownMins := policy.NetSustainCooldownMins
		if cooldownMins <= 0 {
			cooldownMins = 30
		}
		window := time.Duration(windowMins) * time.Minute
		cooldown := time.Duration(cooldownMins) * time.Minute

		snaps, err := store.Load(c.ID, now.Add(-window))
		if err != nil {
			fmt.Fprintf(logErr, "net-alert worker: %s: %v\n", c.Name, err)
			continue
		}
		if len(snaps) == 0 {
			botNetSustainAlertState.Lock()
			delete(botNetSustainAlertState.active, c.ID)
			botNetSustainAlertState.Unlock()
			continue
		}

		ifaces := sustainedCandidateIfaces(snaps, policy)
		if len(ifaces) == 0 {
			continue
		}

		seenKeys := make(map[string]struct{}, len(ifaces))
		minCoverage := time.Duration(float64(window) * 0.9)
		for _, iface := range ifaces {
			key := c.ID + "|" + iface
			seenKeys[key] = struct{}{}

			series := history.AggregateNodeSeries(snaps, iface)
			if len(series) == 0 {
				botNetSustainAlertState.Lock()
				delete(botNetSustainAlertState.active, key)
				delete(botNetSustainAlertState.lastSent, key)
				botNetSustainAlertState.Unlock()
				continue
			}

			coverage := series[len(series)-1].Timestamp.Sub(series[0].Timestamp)
			if coverage < minCoverage {
				continue
			}

			minV := series[0].TotalMbps
			maxV := series[0].TotalMbps
			sumV := 0.0
			triggered := true
			for _, p := range series {
				v := p.TotalMbps
				sumV += v
				if v < minV {
					minV = v
				}
				if v > maxV {
					maxV = v
				}
				if v < policy.NetSustainMbps {
					triggered = false
				}
			}

			botNetSustainAlertState.Lock()
			prevActive := botNetSustainAlertState.active[key]
			lastSent := botNetSustainAlertState.lastSent[key]
			if !triggered {
				botNetSustainAlertState.active[key] = false
				botNetSustainAlertState.Unlock()
				continue
			}
			if prevActive && !lastSent.IsZero() && now.Sub(lastSent) < cooldown {
				botNetSustainAlertState.Unlock()
				continue
			}
			botNetSustainAlertState.active[key] = true
			botNetSustainAlertState.lastSent[key] = now
			botNetSustainAlertState.Unlock()

			avgV := sumV / float64(len(series))
			p95 := history.PercentileMbps(series, 95)
			msg := "🔴 <b>Sustained network alert (CRITICAL)</b>\n" +
				"<b>Cluster:</b> " + htmlEscapeTelegram(c.Name) + "\n" +
				"<b>Interface:</b> " + htmlEscapeTelegram(iface) + "\n" +
				fmt.Sprintf("<b>Condition:</b> ≥ %.1f Mbps for %d min\n", policy.NetSustainMbps, windowMins) +
				fmt.Sprintf("<b>Observed:</b> min %.1f · avg %.1f · p95 %.1f · max %.1f Mbps (%d samples)",
					minV, avgV, p95, maxV, len(series))
			for _, id := range userIDs {
				_ = telegramSendHTML(ctx, client, token, id, msg)
			}
		}

		botNetSustainAlertState.Lock()
		for k := range botNetSustainAlertState.active {
			if !strings.HasPrefix(k, c.ID+"|") {
				continue
			}
			if _, ok := seenKeys[k]; ok {
				continue
			}
			delete(botNetSustainAlertState.active, k)
			delete(botNetSustainAlertState.lastSent, k)
		}
		botNetSustainAlertState.Unlock()
	}
}

func sustainedCandidateIfaces(snaps []history.NetworkSnapshot, policy cluster.AlertPolicy) []string {
	seen := map[string]struct{}{}
	out := make([]string, 0, 16)
	pinned := strings.TrimSpace(policy.NetSustainIface)
	if pinned != "" {
		return []string{pinned}
	}
	for _, snap := range snaps {
		for _, s := range snap.Interfaces {
			iface := strings.TrimSpace(s.Interface)
			if iface == "" {
				continue
			}
			if _, ok := seen[iface]; ok {
				continue
			}
			if !ifaceAllowedByFilters(iface, policy.NetSustainInclude, policy.NetSustainExclude) {
				continue
			}
			seen[iface] = struct{}{}
			out = append(out, iface)
		}
	}
	sort.Strings(out)
	return out
}

func ifaceAllowedByFilters(iface string, include, exclude []string) bool {
	n := strings.ToLower(strings.TrimSpace(iface))
	if n == "" {
		return false
	}
	if len(include) > 0 {
		matched := false
		for _, tok := range include {
			t := strings.ToLower(strings.TrimSpace(tok))
			if t == "" {
				continue
			}
			if strings.Contains(n, t) {
				matched = true
				break
			}
		}
		if !matched {
			return false
		}
	}
	for _, tok := range exclude {
		t := strings.ToLower(strings.TrimSpace(tok))
		if t == "" {
			continue
		}
		if strings.Contains(n, t) {
			return false
		}
	}
	return true
}

func telegramGetUpdates(ctx context.Context, client *http.Client, token string, offset int64, timeoutSeconds int) ([]tgUpdate, int64, error) {
	if timeoutSeconds < 5 {
		timeoutSeconds = 5
	}

	values := url.Values{}
	if offset > 0 {
		values.Set("offset", strconv.FormatInt(offset, 10))
	}
	values.Set("timeout", strconv.Itoa(timeoutSeconds))
	values.Set("allowed_updates", `["message"]`)

	endpoint := "https://api.telegram.org/bot" + token + "/getUpdates?" + values.Encode()
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
	if err != nil {
		return nil, offset, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, offset, err
	}
	defer resp.Body.Close()

	var payload tgUpdatesResponse
	if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
		return nil, offset, err
	}
	if !payload.OK {
		msg := strings.TrimSpace(payload.Description)
		if msg == "" {
			msg = "telegram getUpdates returned ok=false"
		}
		return nil, offset, errors.New(msg)
	}

	next := offset
	for _, upd := range payload.Result {
		if upd.UpdateID >= next {
			next = upd.UpdateID + 1
		}
	}
	return payload.Result, next, nil
}

func telegramSendText(ctx context.Context, client *http.Client, token string, chatID int64, text string) error {
	text = strings.TrimSpace(text)
	if text == "" {
		return nil
	}

	chunks := splitTelegramText(text, 3500)
	for _, chunk := range chunks {
		values := url.Values{}
		values.Set("chat_id", strconv.FormatInt(chatID, 10))
		values.Set("text", chunk)

		endpoint := "https://api.telegram.org/bot" + token + "/sendMessage"
		req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode()))
		if err != nil {
			return err
		}
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

		resp, err := client.Do(req)
		if err != nil {
			return err
		}
		_ = resp.Body.Close()
	}
	return nil
}

func telegramSendHTML(ctx context.Context, client *http.Client, token string, chatID int64, html string) error {
	html = strings.TrimSpace(html)
	if html == "" {
		return nil
	}
	values := url.Values{}
	values.Set("chat_id", strconv.FormatInt(chatID, 10))
	values.Set("text", html)
	values.Set("parse_mode", "HTML")
	values.Set("disable_web_page_preview", "true")

	endpoint := "https://api.telegram.org/bot" + token + "/sendMessage"
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode()))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	_ = resp.Body.Close()
	return nil
}

func htmlEscapeTelegram(s string) string {
	s = strings.ReplaceAll(s, "&", "&amp;")
	s = strings.ReplaceAll(s, "<", "&lt;")
	s = strings.ReplaceAll(s, ">", "&gt;")
	return s
}

func formatBotHelpHTML() string {
	return strings.Join([]string{
		"✻ <b>PXmon (Phylex Monitor) Bot</b>",
		"<i>Secure remote control for your cluster fleet.</i>",
		"",
		"<b>━━ Clusters</b>",
		"• <code>cluster list</code>",
		"• <code>cluster show eu-1</code>",
		"• <code>cluster ping eu-1 --agent</code>",
		"• <code>cluster stats eu-1</code>",
		"",
		"<b>━━ Usage &amp; billing</b>",
		"• <code>usage eu-1</code>  — live processes, RAM, folders, top iface",
		"• <code>traffic eu-1 1h|1d|1mo|all</code>  — P95 text summary",
		"• <code>graph eu-1 1d</code>  — PNG network graph with P95 line",
		"• <code>p95 eu-1 eth0 30d</code>  — interface P95 + graph attachment",
		"",
		"<b>━━ Alerts</b>",
		"• <code>alerts set eu-1 --net-mbps 300 --ram 90 --disk 90</code>",
		"• <code>alerts set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup</code>",
		"• <code>alert-vm check eu-1</code>",
		"",
		"<b>━━ Virtualization</b>",
		"• <code>kvm list --cluster eu-1</code>",
		"• <code>lxd top --cluster eu-1</code>",
		"",
		"<b>━━ Shortcuts</b>",
		"<code>/help</code>  ·  <code>/start</code>",
	}, "\n")
}

func formatBotAccessDeniedHTML(userID int64) string {
	idRepr := "unknown"
	if userID > 0 {
		idRepr = strconv.FormatInt(userID, 10)
	}
	return strings.Join([]string{
		"🔒 <b>Access denied</b>",
		fmt.Sprintf("Your Telegram ID <code>%s</code> is not on the allow-list.", idRepr),
		"",
		"<i>Ask the cluster administrator to add it via</i>",
		"<code>pxmon bot telegram allow &lt;id&gt;</code>",
	}, "\n")
}

func pollAndSendVMAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) {
	clusters, _, err := svc.List()
	if err != nil {
		return
	}
	for _, c := range clusters {
		p, err := svc.GetVMAlertPolicy(c.ID)
		if err != nil || !p.Enabled {
			continue
		}
		routing, _ := svc.GetAlertRouting(c.ID)
		trigger, _ := svc.GetRunbookTrigger(c.ID)
		callCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
		rep, err := svc.CheckVMAlerts(callCtx, c.ID)
		cancel()
		if err != nil {
			fmt.Fprintf(logErr, "vm-alert worker: %s: %v\n", c.Name, err)
			continue
		}
		state := strings.Join(rep.Warnings, "|")
		botVMAlertState.Lock()
		prev := botVMAlertState.last[c.ID]
		lastWarn := botVMAlertState.lastWarnSent[c.ID]
		if state == "" {
			delete(botVMAlertState.last, c.ID)
			delete(botVMAlertState.lastWarnSent, c.ID)
		} else {
			botVMAlertState.last[c.ID] = state
		}
		botVMAlertState.Unlock()
		if state == "" || state == prev {
			continue
		}
		isCritical := false
		for _, w := range rep.Warnings {
			if strings.Contains(strings.ToLower(w), "min_running") || strings.Contains(strings.ToLower(w), "below") {
				isCritical = true
				break
			}
		}
		if !isCritical {
			batchDur := time.Duration(routing.WarningBatchMins) * time.Minute
			if batchDur <= 0 {
				batchDur = 5 * time.Minute
			}
			if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur {
				continue
			}
			botVMAlertState.Lock()
			botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC()
			botVMAlertState.Unlock()
		} else if !routing.CriticalImmediate {
			// If critical-immediate is disabled, still route as warning batch.
			batchDur := time.Duration(routing.WarningBatchMins) * time.Minute
			if batchDur <= 0 {
				batchDur = 5 * time.Minute
			}
			if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur {
				continue
			}
			botVMAlertState.Lock()
			botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC()
			botVMAlertState.Unlock()
		}
		msg := "⚠️ <b>VM alerts</b>\n" +
			"<b>Cluster:</b> " + htmlEscapeTelegram(c.Name) + "\n" +
			fmt.Sprintf("<b>Running:</b> %d  <b>Shut off:</b> %d\n", rep.Running, rep.ShutOff)
		if len(rep.ShutOffNames) > 0 {
			msg += "<b>Shut off VM:</b> " + htmlEscapeTelegram(strings.Join(rep.ShutOffNames, ", ")) + "\n"
		}
		for _, w := range rep.Warnings {
			msg += "• " + htmlEscapeTelegram(w) + "\n"
		}
		for _, id := range userIDs {
			_ = telegramSendHTML(ctx, client, token, id, msg)
		}
		// Auto-trigger runbook on VM shutoff alert.
		if trigger.Enabled && trigger.OnVMShutoff && rep.ShutOff > 0 && strings.TrimSpace(trigger.RunbookID) != "" {
			cooldown := time.Duration(trigger.CooldownMins) * time.Minute
			if cooldown <= 0 {
				cooldown = 30 * time.Minute
			}
			if trigger.LastTriggered.IsZero() || time.Since(trigger.LastTriggered) >= cooldown {
				if err := executeRunbookByID(svc, trigger.RunbookID); err != nil {
					fmt.Fprintf(logErr, "runbook trigger: %s: %v\n", c.Name, err)
				} else {
					_ = svc.TouchRunbookTrigger(c.ID, time.Now().UTC())
					for _, id := range userIDs {
						_ = telegramSendHTML(ctx, client, token, id,
							"🟠 <b>runbook auto-triggered</b>\n"+
								"<b>Cluster:</b> "+htmlEscapeTelegram(c.Name)+"\n"+
								"<b>Runbook:</b> "+htmlEscapeTelegram(trigger.RunbookID))
					}
				}
			}
		}
	}
}

func executeRunbookByID(svc *cluster.Service, runbookID string) error {
	rb, ok := svc.GetRunbook(runbookID)
	if !ok {
		return errors.New("runbook not found: " + runbookID)
	}
	for _, st := range rb.Steps {
		cmd := strings.TrimSpace(st.Command)
		if cmd == "" {
			continue
		}
		_, code := runObserverScopedCommand(svc, svc.ConfigPath(), cmd, observerCommandOptions{
			AllowShellEscape: false,
			StatsAutoOnce:    true,
			BlockBotRun:      true,
			StripANSI:        true,
		})
		if code != 0 {
			return errors.New("step failed: " + st.Title)
		}
	}
	return nil
}

func formatBotCommandReplyHTML(cmd, out string, code int) []string {
	cmd = strings.TrimSpace(cmd)
	body := strings.TrimRight(out, "\n")

	var header string
	if code == 0 {
		header = fmt.Sprintf("🟢 <b>%s</b>", htmlEscapeTelegram(cmd))
	} else {
		header = fmt.Sprintf("🔴 <b>%s</b>  <i>exit %d</i>", htmlEscapeTelegram(cmd), code)
	}

	if strings.TrimSpace(body) == "" {
		if code == 0 {
			return []string{header + "\n<pre>ok</pre>"}
		}
		return []string{header + "\n<pre>(no output)</pre>"}
	}

	bodyChunks := splitTelegramText(htmlEscapeTelegram(body), 3600)
	msgs := make([]string, 0, len(bodyChunks))
	for i, chunk := range bodyChunks {
		if i == 0 {
			msgs = append(msgs, header+"\n<pre>"+chunk+"</pre>")
		} else {
			msgs = append(msgs, "<pre>"+chunk+"</pre>")
		}
	}
	return msgs
}

func splitTelegramText(text string, limit int) []string {
	if limit < 256 {
		limit = 256
	}
	if len(text) <= limit {
		return []string{text}
	}
	lines := strings.Split(text, "\n")
	out := make([]string, 0, len(lines))
	var cur strings.Builder
	for _, line := range lines {
		candidate := line
		if cur.Len() > 0 {
			candidate = "\n" + line
		}
		if cur.Len()+len(candidate) > limit {
			if cur.Len() > 0 {
				out = append(out, cur.String())
				cur.Reset()
			}
			for len(line) > limit {
				out = append(out, line[:limit])
				line = line[limit:]
			}
			if line != "" {
				cur.WriteString(line)
			}
			continue
		}
		cur.WriteString(candidate)
	}
	if cur.Len() > 0 {
		out = append(out, cur.String())
	}
	return out
}

func normalizeTelegramCommand(text string) string {
	text = strings.TrimSpace(text)
	if text == "" {
		return ""
	}
	if strings.HasPrefix(text, "/") {
		text = strings.TrimPrefix(text, "/")
		parts := strings.Fields(text)
		if len(parts) == 0 {
			return ""
		}
		parts[0] = strings.Split(parts[0], "@")[0]
		text = strings.Join(parts, " ")
	}
	if strings.EqualFold(text, "start") {
		return "help"
	}
	if strings.HasPrefix(strings.ToLower(text), "run ") {
		return strings.TrimSpace(text[4:])
	}
	return text
}

func parseInt64CSV(raw string) ([]int64, error) {
	raw = strings.TrimSpace(raw)
	if raw == "" {
		return nil, nil
	}
	parts := strings.Split(raw, ",")
	out := make([]int64, 0, len(parts))
	for _, p := range parts {
		p = strings.TrimSpace(p)
		if p == "" {
			continue
		}
		v, err := strconv.ParseInt(p, 10, 64)
		if err != nil {
			return nil, fmt.Errorf("invalid telegram user id %q", p)
		}
		out = append(out, v)
	}
	return out, nil
}

func formatInt64IDs(ids []int64) string {
	if len(ids) == 0 {
		return "(none)"
	}
	c := append([]int64(nil), ids...)
	sort.Slice(c, func(i, j int) bool { return c[i] < c[j] })
	parts := make([]string, 0, len(c))
	for _, id := range c {
		parts = append(parts, strconv.FormatInt(id, 10))
	}
	return strings.Join(parts, ",")
}

func normalizeInt64IDs(ids []int64) []int64 {
	if len(ids) == 0 {
		return nil
	}
	seen := make(map[int64]struct{}, len(ids))
	out := make([]int64, 0, len(ids))
	for _, id := range ids {
		if id <= 0 {
			continue
		}
		if _, ok := seen[id]; ok {
			continue
		}
		seen[id] = struct{}{}
		out = append(out, id)
	}
	sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
	if len(out) == 0 {
		return nil
	}
	return out
}

type int64SliceFlag []int64

func (s *int64SliceFlag) String() string {
	return formatInt64IDs(*s)
}

func (s *int64SliceFlag) Set(v string) error {
	id, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64)
	if err != nil {
		return fmt.Errorf("invalid user id %q", v)
	}
	*s = append(*s, id)
	return nil
}