From 6b703db02bbf6e184dea11c2360406b94c77c0fe1907bc0fd008f6aad172aba2 Mon Sep 17 00:00:00 2001 From: endssh Date: Tue, 16 Jun 2026 21:52:10 +0400 Subject: [PATCH] chore: publish pxmon v0.2.0 --- .gitattributes | 2 + .github/ISSUE_TEMPLATE/bug_report.md | 23 + .github/ISSUE_TEMPLATE/feature_request.md | 12 + .gitignore | 47 + CHANGELOG.md | 40 + CODE_OF_CONDUCT.md | 29 + CONTRIBUTING.md | 30 + LICENSE | 21 + Makefile | 20 + README.md | 81 + SECURITY.md | 19 + VERSION | 1 + docs/ARCHITECTURE.md | 29 + docs/COMMANDS.md | 119 + docs/FEATURES.md | 67 + docs/INSTALL.md | 41 + docs/PUBLISH.md | 28 + docs/RELEASES.md | 30 + docs/REPOTUNNELING.md | 103 + docs/TUI.md | 40 + docs/USAGE.md | 92 + go.mod | 51 + go.sum | 160 + internal/agent/server.go | 763 +++ internal/agent/statfs_unix.go | 13 + internal/agent/statfs_windows.go | 10 + internal/agent/top.go | 431 ++ internal/cli/app.go | 2969 +++++++++++ internal/cli/bot.go | 1523 ++++++ internal/cli/bot_usage.go | 440 ++ internal/cli/command_exec.go | 182 + internal/cli/extras.go | 2068 ++++++++ internal/cli/monitor.go | 5508 +++++++++++++++++++++ internal/cli/monitor_live.go | 497 ++ internal/cli/monitor_livecmd.go | 306 ++ internal/cli/monitor_privacy.go | 85 + internal/cli/monitor_ssh.go | 466 ++ internal/cli/monitor_usage.go | 241 + internal/cli/shell.go | 286 ++ internal/cli/usage.go | 414 ++ internal/cli/winch_unix.go | 41 + internal/cli/winch_windows.go | 19 + internal/cluster/agent_auth.go | 42 + internal/cluster/agent_versions.go | 68 + internal/cluster/agent_versions.json | 37 + internal/cluster/backup.go | 655 +++ internal/cluster/change_history.go | 89 + internal/cluster/drift.go | 86 + internal/cluster/export.go | 456 ++ internal/cluster/model.go | 518 ++ internal/cluster/p95.go | 82 + internal/cluster/policies.go | 152 + internal/cluster/repo_tunnel.go | 413 ++ internal/cluster/runbook.go | 187 + internal/cluster/scheduler.go | 301 ++ internal/cluster/service.go | 2602 ++++++++++ internal/cluster/service_test.go | 554 +++ internal/cluster/slo_capacity.go | 159 + internal/cluster/store.go | 425 ++ internal/cluster/store_test.go | 110 + internal/cluster/tags.go | 144 + internal/cluster/transport.go | 269 + internal/cluster/usage.go | 272 + internal/cluster/vm_alerts.go | 137 + internal/history/availability_store.go | 92 + internal/history/capacity_store.go | 102 + internal/history/chart.go | 199 + internal/history/network_store.go | 146 + internal/history/percentile.go | 242 + 69 files changed, 25886 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 SECURITY.md create mode 100644 VERSION create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/COMMANDS.md create mode 100644 docs/FEATURES.md create mode 100644 docs/INSTALL.md create mode 100644 docs/PUBLISH.md create mode 100644 docs/RELEASES.md create mode 100644 docs/REPOTUNNELING.md create mode 100644 docs/TUI.md create mode 100644 docs/USAGE.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/agent/server.go create mode 100644 internal/agent/statfs_unix.go create mode 100644 internal/agent/statfs_windows.go create mode 100644 internal/agent/top.go create mode 100644 internal/cli/app.go create mode 100644 internal/cli/bot.go create mode 100644 internal/cli/bot_usage.go create mode 100644 internal/cli/command_exec.go create mode 100644 internal/cli/extras.go create mode 100644 internal/cli/monitor.go create mode 100644 internal/cli/monitor_live.go create mode 100644 internal/cli/monitor_livecmd.go create mode 100644 internal/cli/monitor_privacy.go create mode 100644 internal/cli/monitor_ssh.go create mode 100644 internal/cli/monitor_usage.go create mode 100644 internal/cli/shell.go create mode 100644 internal/cli/usage.go create mode 100644 internal/cli/winch_unix.go create mode 100644 internal/cli/winch_windows.go create mode 100644 internal/cluster/agent_auth.go create mode 100644 internal/cluster/agent_versions.go create mode 100644 internal/cluster/agent_versions.json create mode 100644 internal/cluster/backup.go create mode 100644 internal/cluster/change_history.go create mode 100644 internal/cluster/drift.go create mode 100644 internal/cluster/export.go create mode 100644 internal/cluster/model.go create mode 100644 internal/cluster/p95.go create mode 100644 internal/cluster/policies.go create mode 100644 internal/cluster/repo_tunnel.go create mode 100644 internal/cluster/runbook.go create mode 100644 internal/cluster/scheduler.go create mode 100644 internal/cluster/service.go create mode 100644 internal/cluster/service_test.go create mode 100644 internal/cluster/slo_capacity.go create mode 100644 internal/cluster/store.go create mode 100644 internal/cluster/store_test.go create mode 100644 internal/cluster/tags.go create mode 100644 internal/cluster/transport.go create mode 100644 internal/cluster/usage.go create mode 100644 internal/cluster/vm_alerts.go create mode 100644 internal/history/availability_store.go create mode 100644 internal/history/capacity_store.go create mode 100644 internal/history/chart.go create mode 100644 internal/history/network_store.go create mode 100644 internal/history/percentile.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b320047 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto eol=lf +*.bat text eol=crlf diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..55f06d3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +about: Report a reproducible problem +--- + +## Summary + +## Steps to reproduce +1. +2. +3. + +## Expected behavior + +## Actual behavior + +## Environment +- OS: +- Go version: +- PXmon version: +- Command used: + +## Logs / output diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..68dacf8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,12 @@ +--- +name: Feature request +about: Suggest a new capability or improvement +--- + +## Problem statement + +## Proposed solution + +## Alternatives considered + +## Additional context diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3903cc8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,47 @@ +# Binaries +pxmon +pxmon-agent +observerctl +observer-agent +/bin/ +/dist/ + +# Build/test artifacts +*.test +*.out +*.cover +coverage.* +*.prof + +# OS/editor noise +.DS_Store +.idea/ +.vscode/ +*.swp +*.swo + +# Runtime/config secrets +*.enc +*.key +*.pem +*.p12 +*.crt +*.log +report.json +*.pid + +# Local env files +.env +.env.* + +# Temporary files +/tmp/ +*.tmp +*.bak +*~ + +# Generated assets +*.png +*.jpg +*.jpeg +*.gif diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..3d88634 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on Keep a Changelog and this project follows Semantic Versioning. + +## [0.2.0] - 2026-06-16 +### Added +- `cluster exec` / `cluster run` command path for direct remote shell execution. +- RepoTunneling commands for controlled package repository access through a gateway proxy. +- `--key-passphrase-file` and `--store-key-passphrase-file` support for SSH key workflows. +- Export/import support for referenced SSH/SFTP key files, restored under the destination config directory. +- Backup target merge behavior during config import. +- Known agent version metadata for `v0.2.0`. +- RepoTunneling documentation. + +### Changed +- Agent signed request timestamp tolerance increased to five minutes. +- TUI refresh defaults to a calmer two-second interval and avoids overlapping refreshes. +- Network dashboard rows now enrich historical stats lazily for smoother rendering. +- Cluster list JSON now includes repo tunnel state. +- README and release docs now point to the Gitea public repository. + +### Fixed +- Windows builds no longer fail on Unix-only `statfs` calls. +- Open-source package excludes local runtime data and release binaries from the source tree. + +## [0.1.0] - 2026-04-17 +### Added +- First public open-source package layout for PXmon. +- Interactive Bubble Tea TUI for overview, clusters, network, settings, docs, usage, and live views. +- SSH cluster registry with encrypted local storage. +- `pxmon-agent` bootstrap/update flow and agent status/version checks. +- Alerting, VM alert policies, runbooks, schedules, backup plans, drift checks, SLO and capacity reports. +- Telegram bot integration commands and daemon controls. +- Config export/import encrypted bundles. + +### Notes +- This release is aligned with the current workspace state on 2026-04-17. +- Future releases should be added above this section. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..752822a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,29 @@ +# Code of Conduct + +## Our pledge +We as contributors and maintainers pledge to make participation in this project +an open, welcoming, and harassment-free experience for everyone. + +## Our standards +Examples of behavior that contributes to a positive environment include: +- Using welcoming and respectful language +- Being constructive during technical disagreements +- Accepting and giving actionable feedback + +Examples of unacceptable behavior include: +- Harassment or discriminatory language +- Personal attacks, trolling, or insulting comments +- Publishing private information without permission + +## Enforcement +Project maintainers are responsible for clarifying and enforcing standards of +acceptable behavior and may remove, edit, or reject comments, commits, code, +or other contributions that are not aligned with this Code of Conduct. + +## Scope +This Code of Conduct applies in project spaces and in public spaces when an +individual is representing the project. + +## Reporting +Report violations to project maintainers through private channels defined in +`SECURITY.md`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..89595e2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# Contributing + +## Development setup +1. Install Go 1.25+. +2. Clone repository. +3. Run: + - `go mod tidy` + - `go test ./...` + +## Build locally +- `go build -o bin/pxmon ./cmd/pxmon` +- `go build -o bin/pxmon-agent ./cmd/pxmon-agent` + +## Code style +- Keep changes small and focused. +- Prefer explicit error messages with command context. +- Add tests for behavior changes when possible. + +## Commit style +Use clear, scoped commit messages, for example: +- `feat(cluster): add alert routing batch window` +- `fix(tui): stabilize panel heights during scroll` +- `docs: expand command reference` + +## Pull requests +Include: +- What changed +- Why it changed +- How you tested it +- Any behavior or migration impact diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c437f0d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 PXmon Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3962172 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +APP=pxmon +AGENT=pxmon-agent + +.PHONY: build test tidy run clean + +build: + go build -o bin/$(APP) ./cmd/$(APP) + go build -o bin/$(AGENT) ./cmd/$(AGENT) + +test: + go test ./... + +tidy: + go mod tidy + +run: build + ./bin/$(APP) + +clean: + rm -rf bin/$(APP) bin/$(AGENT) diff --git a/README.md b/README.md new file mode 100644 index 0000000..049e82d --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# PXmon + +> Current release: v0.2.0. +> The original GitHub repository is archived; active public source now lives on Gitea. + +PXmon (Phylex Monitor) is a terminal-first SSH cluster manager and monitoring tool with a built-in TUI, agent bootstrap flow, alerting, reporting, backups, and runbook/scheduler automation. + +## Highlights +- Bubble Tea dashboard (`overview`, `clusters`, `network`, `settings`, `docs`, `usage`, `live`) +- Secure encrypted local cluster registry +- SSH and optional SSH-tunneled agent transport (`direct` / `ipfabric`) +- Remote command execution and embedded SSH workflows +- Agent bootstrap, status checks, version checks, and updates +- Alerts (CPU/RAM/SWAP/DISK/NET), VM alerts, and alert routing +- Drift detection, SLO reporting, capacity forecast, and traffic p95 +- Backup targets/plans (SFTP/S3), runbooks, and scheduled tasks +- RepoTunneling for temporary package repository access through a controlled gateway +- Telegram bot controls +- Encrypted export/import bundles, including referenced key files + +## Requirements +- Go `1.25+` +- Linux/macOS terminal (interactive features require TTY) +- SSH reachability to target nodes + +## Quick start +```bash +git clone https://git.datacoria.com/endssh/pxmon.git +cd pxmon + +go build -o bin/pxmon ./cmd/pxmon +go build -o bin/pxmon-agent ./cmd/pxmon-agent + +./bin/pxmon +``` + +Or run by command: +```bash +./bin/pxmon cluster help +./bin/pxmon tui +``` + +## Core commands +- `pxmon cluster ...` +- `pxmon tui` +- `pxmon clusters` +- `pxmon network` +- `pxmon cluster exec ...` +- `pxmon cluster repo-tunnel ...` +- `pxmon bot telegram ...` +- `pxmon locker ...` +- `pxmon config export|import ...` +- `pxmon explain --find ` + +## Documentation +- [Install Guide](docs/INSTALL.md) +- [Usage Guide](docs/USAGE.md) +- [Command Reference](docs/COMMANDS.md) +- [TUI Guide](docs/TUI.md) +- [Architecture](docs/ARCHITECTURE.md) +- [Feature Overview](docs/FEATURES.md) +- [RepoTunneling](docs/REPOTUNNELING.md) +- [Releases and Versioning](docs/RELEASES.md) +- [Publish Guide](docs/PUBLISH.md) + +## Default config location +By default PXmon stores encrypted registry data in: +- Linux: `~/.config/pxmon/clusters.enc` +- macOS: `~/Library/Application Support/pxmon/clusters.enc` + +You can override using: +- `PXMON_CONFIG` +- `PXMON_MASTER_KEY` +- `--config /path/to/clusters.enc` + +## Open source metadata +- License: [MIT](LICENSE) +- Version: [`VERSION`](VERSION) +- Changelog: [CHANGELOG.md](CHANGELOG.md) +- Security: [SECURITY.md](SECURITY.md) +- Contributing: [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..dc08e8a --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,19 @@ +# Security Policy + +## Supported versions +Security fixes are applied to the latest release branch and `main`. + +## Reporting a vulnerability +Do not open public issues for security vulnerabilities. + +Until a dedicated private inbox is configured, report privately to repository +maintainers and include: +- Impact summary +- Reproduction steps +- Affected versions or commit hash +- Suggested mitigation (if available) + +## Secret handling guidance +- Never commit real passwords, private keys, tokens, or host fingerprints. +- Use `--store-password` and key passphrase flags only in local trusted setups. +- Treat exported encrypted bundles (`*.enc`) as sensitive artifacts. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..a82804c --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2.0 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..a15bbe6 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,29 @@ +# Architecture + +## Components +- `cmd/pxmon`: main CLI and TUI entrypoint. +- `cmd/pxmon-agent`: node-side agent binary. +- `internal/cli`: command parsing, TUI, bot integration, UX flows. +- `internal/cluster`: cluster model/store/service, SSH operations, alerts, reports, backup/runbooks/scheduler. +- `internal/history`: metrics history stores and chart helpers. +- `internal/agent`: agent API server and stats collection types. + +## Data model +- Registry is encrypted and stored locally. +- Default path: `/pxmon/clusters.enc` +- Master key path defaults to `/pxmon/master.key` +- Supports environment overrides: + - `PXMON_CONFIG` + - `PXMON_MASTER_KEY` + +## Runtime flow +1. User invokes CLI/TUI command. +2. `internal/cli` resolves cluster selector and command flags. +3. `internal/cluster.Service` performs SSH/agent calls. +4. Results are rendered in CLI tables/JSON or TUI panels. +5. Optional telemetry/history snapshots are persisted for trends. + +## Security model +- Local registry encryption with a generated master key. +- Locker subsystem can block operational commands until unlock. +- Config export/import supports passphrase-protected bundles. diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md new file mode 100644 index 0000000..b0bf258 --- /dev/null +++ b/docs/COMMANDS.md @@ -0,0 +1,119 @@ +# Command Reference + +## Root +- `pxmon [--config path] [--json] ` +- `pxmon help` +- `pxmon shell` (interactive alias to TUI) +- `pxmon tui [name-or-id] [--interval 2s] [--iface eth0]` +- `pxmon clusters [name-or-id] [--interval 2s] [--iface eth0]` +- `pxmon network [name-or-id] [--interval 2s] [--iface eth0]` +- `pxmon cluster ...` +- `pxmon bot ...` +- `pxmon locker ...` +- `pxmon config ...` +- `pxmon export ...` (shortcut) +- `pxmon import ...` (shortcut) +- `pxmon explain [--find text]` + +## Cluster command tree +- `connect|add` +- `list|ls` +- `show|get` +- `current` +- `use` +- `disconnect|remove|rm` +- `set-auth|auth|password|passwd` +- `openssh|ssh` +- `exec|run -- ` +- `ping|check` +- `bootstrap` +- `agent` + - `status [name-or-id]` + - `adopt-auth|sync-auth|repair-auth [name-or-id]` + - `update [name-or-id] [--restart-bot=true|false]` + - `versions|version-list` +- `stats [name-or-id] [--once]` +- `usage [name-or-id] --range live|1h|1d|1mo|all [--du /]` +- `traffic [name-or-id] --range 1h|1d|1mo|all` +- `graph [name-or-id] --range 1h|1d|1mo|all [--out file.png]` +- `p95 [name-or-id] --iface --range [--graph]` +- `slo|availability [name-or-id] --range 7d|30d|all` +- `capacity forecast [name-or-id] --range 7d|30d|all` +- `alert|alerts show|set [name-or-id]` +- `alert-routing|routing show|set [name-or-id]` +- `alert-vm|vm-alert show|set|check [name-or-id]` +- `software|plugins show|scan [name-or-id]` +- `tag|tags add|rm|ls [name-or-id] --tags a,b` +- `kvm-tag|vm-tag add|rm|ls [name-or-id] --vm --tags a,b` +- `change-history|changes [--tail N]` +- `drift [name-or-id]` + - `baseline set|show [name-or-id]` + - `ack [name-or-id] --kind --for 24h` +- `report export --format json|csv --out ` +- `backup` + - `target add|ls|rm|test` + - `plan add|ls|rm|run` + - `run ` +- `repo-tunnel|repo|repo-tunneling` + - `gateway-script --allow [--port 3128]` + - `gateway-setup --allow [--port 3128]` + - `enable --gateway --table [--gateway-ip ] [--manager auto|apt|dnf|yum]` + - `install --gateway --table -- ` + - `status ` + - `disable --gateway --table ` +- `runbook` + - `list` + - `show ` + - `run ` + - `add --id ... --name ... --step ...` + - `add --edit` + - `add --from ` + - `rm ` +- `runbook-trigger show|set [name-or-id]` +- `schedule|scheduler` + - `add --name ... --cmd ... --every ...` + - `add --edit` + - `add --from ` + - `ls` + - `rm ` + - `run-due` + - `start [--interval 30s]` + - `stop` + - `status` + - `logs [--tail 200]` + - `worker [--interval 30s]` + +## Bot +- `pxmon bot telegram show [--show-token]` +- `pxmon bot telegram set --token --allow [--allow ...]` +- `pxmon bot telegram disable` +- `pxmon bot telegram restart [--poll 2s]` +- `pxmon bot telegram logs [--tail 200]` +- `pxmon bot telegram run [--poll 2s]` + +## Locker +- `pxmon locker status` +- `pxmon locker set [--password ]` +- `pxmon locker unlock [--password ]` +- `pxmon locker lock` +- `pxmon locker disable` +- `pxmon locker logs [--tail 200]` + +## Config +- `pxmon config export [] [--out ] [--password ]` +- `pxmon config import [--password ] [--replace]` + +## In-TUI command console +Alias examples: +- `alerts ...` -> `cluster alert ...` +- `connect ` -> `cluster use ` +- `software ...` -> `cluster software ...` +- `clusters` -> `cluster list` +- `ping ` -> `cluster ping ` + +Plugin-style commands from console: +- `kvm list|start|stop|reboot|top|net-top` +- `lxc list|start|stop|restart|stats|top|net-top` +- `lxd list|start|stop|restart|stats|top|net-top` +- `bird status|protocols|routes` +- `frr status|routes|bgp|ospf` diff --git a/docs/FEATURES.md b/docs/FEATURES.md new file mode 100644 index 0000000..618781f --- /dev/null +++ b/docs/FEATURES.md @@ -0,0 +1,67 @@ +# Feature Overview + +## 1) Cluster inventory and connectivity +- Register clusters by SSH host/user/auth method. +- Keep one active cluster for short commands. +- Validate connectivity at connect time (or allow deferred validation). +- Support direct agent transport or SSH-tunneled transport (`ipfabric`). +- Store SSH key passphrases directly or via a passphrase file reference. + +## 2) Interactive TUI operations +- Multi-view dashboard for overview, cluster inventory, network, settings, docs, usage, and live data. +- Integrated command console with history and aliases. +- Scrollable fixed-size panes for stable rendering. +- Privacy mode for masking sensitive output. + +## 3) Agent lifecycle +- Build and bootstrap `pxmon-agent` remotely via SSH. +- Query agent status across all clusters. +- Compare node agent version with expected local version. +- Update and optionally restart bot processes. + +## 4) Monitoring and analytics +- Realtime stats snapshots. +- Historical usage windows (`live`, `1h`, `1d`, `1mo`, `all`). +- Traffic p95 and usage chart generation (PNG output). +- SLO reports and capacity forecasting. + +## 5) Alerting and policy controls +- CPU/RAM/SWAP/DISK/NET threshold policies. +- Sustained network threshold detection. +- VM alert policy (e.g., minimum running VMs, shutoff warnings). +- Alert routing behavior controls (critical immediate vs warning batches). + +## 6) Drift and change management +- Drift checks for configuration/runtime mismatches. +- Baseline set/show and timed acknowledgment. +- Change history inspection. + +## 7) Tagging and grouping +- Cluster tags (add/remove/list). +- KVM VM-level tags. + +## 8) Backup and disaster readiness +- Backup targets (SFTP/S3). +- Backup plans with schedules and retention settings. +- Manual backup run execution. +- Export/import bundles can carry referenced SSH/SFTP key files. + +## 9) RepoTunneling +- Prepare a restricted squid gateway for repository traffic. +- Temporarily configure apt/dnf/yum proxy settings on `ipfabric` nodes. +- Run one-shot package install commands and clean up the route/proxy state. + +## 10) Runbooks and automation +- Define and execute runbooks. +- Configure runbook triggers for selected alert conditions. +- Scheduler for periodic command execution. + +## 11) Telegram bot integration +- Configure token and allow-list. +- Run/restart/disable bot daemon. +- Tail bot logs. + +## 12) Security and data management +- Encrypted local registry storage. +- Locker controls to block critical commands until unlock. +- Encrypted export/import bundles for migration and backup. diff --git a/docs/INSTALL.md b/docs/INSTALL.md new file mode 100644 index 0000000..70fc95e --- /dev/null +++ b/docs/INSTALL.md @@ -0,0 +1,41 @@ +# Install and Build + +## Prerequisites +- Go 1.25+ +- Git +- SSH client access to managed hosts + +## Clone and build +```bash +git clone https://git.datacoria.com/endssh/pxmon.git +cd pxmon +go mod tidy +go test ./... +go build -o bin/pxmon ./cmd/pxmon +go build -o bin/pxmon-agent ./cmd/pxmon-agent +``` + +## Run +```bash +./bin/pxmon +``` + +## Build with explicit agent version metadata +`pxmon-agent` exposes build-time version in API responses. + +```bash +go build -ldflags "-X pxmon/internal/agent.Version=v0.2.0" -o bin/pxmon-agent ./cmd/pxmon-agent +``` + +## Optional install to PATH +```bash +install -m 0755 bin/pxmon /usr/local/bin/pxmon +install -m 0755 bin/pxmon-agent /usr/local/bin/pxmon-agent +``` + +## Verify +```bash +pxmon cluster help +pxmon bot telegram --help +pxmon config --help +``` diff --git a/docs/PUBLISH.md b/docs/PUBLISH.md new file mode 100644 index 0000000..3b57c87 --- /dev/null +++ b/docs/PUBLISH.md @@ -0,0 +1,28 @@ +# Publish to Gitea + +## 1) Initialize repository +```bash +git init --object-format=sha256 +git checkout -b main +git add . +git commit -m "chore: publish pxmon v0.2.0" +``` + +## 2) Create remote and push +```bash +git branch -M main +git remote add origin https://git.datacoria.com/endssh/pxmon.git +git push -u origin main +``` + +## 3) Create release tag +```bash +git tag v0.2.0 +git push origin v0.2.0 +``` + +## 4) Optional repository settings +- Enable branch protection for `main` +- Enable Dependabot alerts +- Add issue labels and project board +- Add release notes for tag `v0.2.0` diff --git a/docs/RELEASES.md b/docs/RELEASES.md new file mode 100644 index 0000000..cdc3291 --- /dev/null +++ b/docs/RELEASES.md @@ -0,0 +1,30 @@ +# Releases and Versioning + +## Versioning policy +This project follows Semantic Versioning: +- `MAJOR`: breaking CLI/API/data changes +- `MINOR`: backward-compatible features +- `PATCH`: backward-compatible fixes + +Current project version is tracked in [`VERSION`](../VERSION). + +## Release process +1. Update `VERSION`. +2. Add new section in `CHANGELOG.md` with date and changes. +3. Run: + - `go test ./...` + - `go build -o bin/pxmon ./cmd/pxmon` + - `go build -o bin/pxmon-agent ./cmd/pxmon-agent` +4. Tag release: + - `git tag vX.Y.Z` + - `git push origin vX.Y.Z` + +## Versions list +- `v0.2.0` (2026-06-16): RepoTunneling, remote exec, key passphrase files, export/import key-file restore, and release cleanup. +- `v0.1.0` (2026-04-17): initial open-source packaging release. + +## Agent versions +Known agent versions are stored in: +- `internal/cluster/agent_versions.json` + +This list is used by agent status/drift checks to flag outdated nodes. diff --git a/docs/REPOTUNNELING.md b/docs/REPOTUNNELING.md new file mode 100644 index 0000000..bca6fc7 --- /dev/null +++ b/docs/REPOTUNNELING.md @@ -0,0 +1,103 @@ +# PXmon RepoTunneling + +RepoTunneling gives an `ipfabric` node temporary package repository access +through a disposable gateway VM. + +The node does not receive full internet access. PXmon only adds an `ip rule` +for the gateway VM address and configures the node package manager to use the +gateway as an HTTP proxy. + +## Topology + +```text +ipfabric node + -> ip rule to gateway public IP using the node-specific routing table + -> gateway VM with squid + -> internet repositories +``` + +The routing table is not hard-coded. Always pass the correct table for that +node with `--table`. + +## 1. Prepare The Gateway VM + +The gateway VM must have normal internet access and must be reachable from the +ipfabric node after adding the destination-specific route rule. + +If the gateway VM is already managed by PXmon: + +```bash +pxmon cluster repo-tunnel gateway-setup repo-vm \ + --allow 198.51.100.20/32 \ + --port 3128 +``` + +If it is not managed by PXmon, print the setup script and run it manually on the +gateway VM: + +```bash +pxmon cluster repo-tunnel gateway-script \ + --allow 198.51.100.20/32 \ + --port 3128 +``` + +Use one `--allow` per node source IP/CIDR. Do not leave squid open to the +internet. + +## 2. Enable RepoTunneling On The ipfabric Node + +```bash +pxmon cluster repo-tunnel enable edge-node-1 \ + --gateway 203.0.113.10:3128 \ + --gateway-ip 203.0.113.10 \ + --table 1010 \ + --manager dnf +``` + +`--gateway-ip` is optional when the node can resolve the gateway hostname or +when `--gateway` is already an IP. It is useful before DNS works through the +proxy path. + +PXmon writes managed package-manager config: + +- apt: `/etc/apt/apt.conf.d/99-pxmon-repo-tunnel` +- dnf: managed block in `/etc/dnf/dnf.conf` +- yum: managed block in `/etc/yum.conf` + +## 3. Install Packages + +For a one-shot package operation: + +```bash +pxmon cluster repo-tunnel install edge-node-1 \ + --gateway 203.0.113.10:3128 \ + --gateway-ip 203.0.113.10 \ + --table 1010 \ + --manager dnf \ + -- dnf install -y curl jq smartmontools +``` + +By default `install` removes the proxy config and matching `ip rule` after the +command. Add `--keep-enabled` if you want to leave it active. + +## 4. Inspect Or Disable + +```bash +pxmon cluster repo-tunnel status edge-node-1 +``` + +```bash +pxmon cluster repo-tunnel disable edge-node-1 \ + --gateway 203.0.113.10:3128 \ + --gateway-ip 203.0.113.10 \ + --table 1010 +``` + +## Notes + +- `--table` is required because ipfabric route table IDs differ between + servers. +- Use `--no-rule` only if you already created the needed route rule manually + and only want PXmon to manage package proxy config. +- For AlmaLinux 8, `--manager dnf` is the expected mode. +- The gateway should be disposable and firewall-restricted. diff --git a/docs/TUI.md b/docs/TUI.md new file mode 100644 index 0000000..806c924 --- /dev/null +++ b/docs/TUI.md @@ -0,0 +1,40 @@ +# TUI Guide + +## Views +- `overview` +- `clusters` +- `network` +- `settings` +- `docs` +- `usage` +- `live` + +## Main hotkeys +- `tab` switch views +- `o/c/n/s/d/u` jump to overview/clusters/network/settings/docs/usage +- `t` open command console +- `ctrl+t` fullscreen command console +- `ctrl+g` leave command console +- `r` refresh +- `q` quit + +## Scrolling +- Main content scroll: `alt+up` / `alt+down` +- Docs view scroll: arrows, `j/k`, `PgUp/PgDn`, `Home/End` +- Network view: navigation with arrows + page controls + +## Console mode +Built-ins: +- `help` +- `history` +- `clear` +- `overview`, `clusters`, `network`, `settings`, `docs` +- `quit` + +Advanced: +- End line with `\` for multiline command continuation +- Shell escape with `!` + +## Privacy mode +- Toggle with `alt+p` +- Redacts sensitive patterns in rendered output diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000..d2f2379 --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,92 @@ +# Usage Guide + +## Start TUI +```bash +pxmon +# or +pxmon tui +``` + +## Add a cluster +```bash +pxmon cluster connect \ + --name eu-1 \ + --host 10.0.0.10 \ + --user root \ + --auth key \ + --key-path ~/.ssh/id_ed25519 +``` + +## Set active cluster +```bash +pxmon cluster use eu-1 +``` + +## Connectivity checks +```bash +pxmon cluster ping eu-1 +pxmon cluster ping eu-1 --agent +``` + +## Agent lifecycle +```bash +pxmon cluster bootstrap eu-1 +pxmon cluster agent status eu-1 +pxmon cluster agent update eu-1 --restart-bot=true +pxmon cluster agent versions +``` + +## Monitoring and history +```bash +pxmon cluster stats eu-1 --once +pxmon cluster usage eu-1 --range 1d +pxmon cluster traffic eu-1 --range 30d +pxmon cluster graph eu-1 --range 1d --out ./eu-1-1d.png +pxmon cluster p95 eu-1 --iface eth0 --range 30d --graph +``` + +## Alerts +```bash +pxmon cluster alert set eu-1 --cpu 85 --ram 90 --disk 90 --net-mbps 300 +pxmon cluster alert-routing set eu-1 --critical-immediate=true --warning-batch-mins 5 +pxmon cluster alert-vm set eu-1 --enabled --warn-on-shutoff --min-running 100 +``` + +## Drift, SLO, capacity +```bash +pxmon cluster drift eu-1 +pxmon cluster drift baseline set eu-1 +pxmon cluster slo eu-1 --range 30d +pxmon cluster capacity forecast eu-1 --range 30d +``` + +## Backup, runbooks, scheduler +```bash +pxmon cluster backup target add --name b2 --type s3 --s3-endpoint s3.example.net --s3-bucket backups --s3-access-key AKIA... --s3-secret-key ... +pxmon cluster backup plan add --name vm-images --cluster eu-1 --target b2 --path /var/lib/libvirt/images --every 6h +pxmon cluster backup run vm-images + +pxmon cluster runbook list +pxmon cluster runbook run vm-health-check + +pxmon cluster schedule add --name audit --cmd 'cluster drift eu-1' --every 30m +pxmon cluster schedule start --interval 30s +``` + +## Telegram bot +```bash +pxmon bot telegram set --token --allow --allow +pxmon bot telegram show +pxmon bot telegram restart +pxmon bot telegram logs --tail 100 +``` + +## Locker and config bundles +```bash +pxmon locker set --password 'strong-pass' +pxmon locker status +pxmon locker unlock --password 'strong-pass' + +pxmon config export --out ./pxmon-export.enc +pxmon config import ./pxmon-export.enc --replace +``` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4861324 --- /dev/null +++ b/go.mod @@ -0,0 +1,51 @@ +module pxmon + +go 1.25.0 + +require ( + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 + github.com/creack/pty v1.1.24 + github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 + github.com/minio/minio-go/v7 v7.0.100 + github.com/pkg/sftp v1.13.10 + github.com/wcharczuk/go-chart/v2 v2.1.2 + golang.org/x/crypto v0.50.0 + golang.org/x/term v0.42.0 +) + +require ( + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/x/ansi v0.10.1 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/image v0.18.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ecd0ecc --- /dev/null +++ b/go.sum @@ -0,0 +1,160 @@ +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw= +github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ= +github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02 h1:AgcIVYPa6XJnU3phs104wLj8l5GEththEw6+F79YsIY= +github.com/hinshun/vt10x v0.0.0-20220301184237-5011da428d02/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8= +github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/wcharczuk/go-chart/v2 v2.1.2 h1:Y17/oYNuXwZg6TFag06qe8sBajwwsuvPiJJXcUcLL6E= +github.com/wcharczuk/go-chart/v2 v2.1.2/go.mod h1:Zi4hbaqlWpYajnXB2K22IUYVXRXaLfSGNNR7P4ukyyQ= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/agent/server.go b/internal/agent/server.go new file mode 100644 index 0000000..c60ca2e --- /dev/null +++ b/internal/agent/server.go @@ -0,0 +1,763 @@ +package agent + +import ( + "bufio" + "context" + "crypto/hmac" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "net/http" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Version is set via ldflags at build time. +var Version = "v0.2.0" + +// Config controls agent runtime. +type Config struct { + ListenAddr string `json:"listen_addr"` + Token string `json:"token"` + RequestSecret string `json:"request_secret,omitempty"` + TLSEnabled bool `json:"tls_enabled,omitempty"` + TLSCertPath string `json:"tls_cert_path,omitempty"` + TLSKeyPath string `json:"tls_key_path,omitempty"` +} + +var ( + cpuSampleMu sync.Mutex + prevCPUTotal uint64 + prevCPUIdle uint64 + prevCPUSet bool +) + +const signedRequestMaxSkewSeconds = 300 + +type nonceState struct { + mu sync.Mutex + seen map[string]int64 +} + +// StatsResponse contains basic node metrics. +type StatsResponse struct { + Timestamp time.Time `json:"timestamp"` + Host HostStats `json:"host"` + CPU CPUStats `json:"cpu"` + Memory MemoryStats `json:"memory"` + Disk []DiskStats `json:"disk"` + Network []NetworkStat `json:"network"` +} + +// HostStats contains host-level facts. +type HostStats struct { + Hostname string `json:"hostname"` + OS string `json:"os"` + Arch string `json:"arch"` + UptimeSeconds float64 `json:"uptime_seconds"` +} + +// CPUStats contains CPU/load values. +type CPUStats struct { + LogicalCores int `json:"logical_cores"` + UsagePercent float64 `json:"usage_percent"` + Load1 float64 `json:"load_1"` + Load5 float64 `json:"load_5"` + Load15 float64 `json:"load_15"` +} + +// MemoryStats contains RAM usage values in bytes. +type MemoryStats struct { + TotalBytes uint64 `json:"total_bytes"` + AvailableBytes uint64 `json:"available_bytes"` + UsedBytes uint64 `json:"used_bytes"` + UsedPercent float64 `json:"used_percent"` + SwapTotalBytes uint64 `json:"swap_total_bytes"` + SwapFreeBytes uint64 `json:"swap_free_bytes"` + SwapUsedBytes uint64 `json:"swap_used_bytes"` + SwapUsedPct float64 `json:"swap_used_percent"` +} + +// DiskStats contains one mount usage entry. +type DiskStats struct { + Source string `json:"source"` + MountPoint string `json:"mount_point"` + FSType string `json:"fs_type"` + Device string `json:"device"` + TotalBytes uint64 `json:"total_bytes"` + FreeBytes uint64 `json:"free_bytes"` + UsedBytes uint64 `json:"used_bytes"` + UsedPercent float64 `json:"used_percent"` + ReadOnly bool `json:"read_only"` + DeviceState string `json:"device_state,omitempty"` + ErrorsCount uint64 `json:"errors_count"` + Health string `json:"health"` + Warnings []string `json:"warnings,omitempty"` +} + +// NetworkStat contains one interface counters snapshot. +type NetworkStat struct { + Interface string `json:"interface"` + RxBytes uint64 `json:"rx_bytes"` + TxBytes uint64 `json:"tx_bytes"` + RxPackets uint64 `json:"rx_packets"` + TxPackets uint64 `json:"tx_packets"` + RxDrops uint64 `json:"rx_drops"` + TxDrops uint64 `json:"tx_drops"` +} + +// LoadConfig reads JSON config from disk. +func LoadConfig(path string) (Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return Config{}, err + } + + var cfg Config + if err := json.Unmarshal(data, &cfg); err != nil { + return Config{}, err + } + + cfg.ListenAddr = strings.TrimSpace(cfg.ListenAddr) + cfg.Token = strings.TrimSpace(cfg.Token) + cfg.RequestSecret = strings.TrimSpace(cfg.RequestSecret) + cfg.TLSCertPath = strings.TrimSpace(cfg.TLSCertPath) + cfg.TLSKeyPath = strings.TrimSpace(cfg.TLSKeyPath) + if cfg.ListenAddr == "" { + cfg.ListenAddr = "0.0.0.0:19090" + } + if cfg.Token == "" { + return Config{}, errors.New("token is required") + } + if cfg.TLSEnabled { + if cfg.TLSCertPath == "" || cfg.TLSKeyPath == "" { + return Config{}, errors.New("tls_enabled=true requires tls_cert_path and tls_key_path") + } + } + return cfg, nil +} + +// Run starts the HTTP API server. +func Run(cfg Config) error { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok\n")) + }) + + mux.Handle("/api/v1/ping", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "status": "ok", + "version": Version, + "time": time.Now().UTC(), + }) + }))) + + mux.Handle("/api/v1/stats", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + stats, err := CollectStats() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, stats) + }))) + + mux.Handle("/api/v1/top", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + q := r.URL.Query() + limit, _ := strconv.Atoi(q.Get("limit")) + sampleMs, _ := strconv.Atoi(q.Get("sample_ms")) + window := time.Duration(sampleMs) * time.Millisecond + top, err := CollectTopProcesses(window, limit) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, top) + }))) + + mux.Handle("/api/v1/du", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + q := r.URL.Query() + root := q.Get("path") + if strings.TrimSpace(root) == "" { + root = "/" + } + limit, _ := strconv.Atoi(q.Get("limit")) + timeoutMs, _ := strconv.Atoi(q.Get("timeout_ms")) + if timeoutMs <= 0 || timeoutMs > 45000 { + timeoutMs = 15000 + } + ctx, cancel := context.WithTimeout(r.Context(), time.Duration(timeoutMs)*time.Millisecond) + defer cancel() + resp, err := CollectDirSizes(ctx, root, limit) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, resp) + }))) + + srv := &http.Server{ + Addr: cfg.ListenAddr, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + } + + if cfg.TLSEnabled { + return srv.ListenAndServeTLS(cfg.TLSCertPath, cfg.TLSKeyPath) + } + return srv.ListenAndServe() +} + +// CollectStats gathers a metrics snapshot. +func CollectStats() (StatsResponse, error) { + hostname, _ := os.Hostname() + + uptime := readUptimeSeconds() + load1, load5, load15 := readLoadAvg() + usage := readCPUUsagePercent() + mem := readMemory() + disk := readDiskStats() + netStats := readNetworkStats() + + stats := StatsResponse{ + Timestamp: time.Now().UTC(), + Host: HostStats{ + Hostname: hostname, + OS: runtime.GOOS, + Arch: runtime.GOARCH, + UptimeSeconds: uptime, + }, + CPU: CPUStats{ + LogicalCores: runtime.NumCPU(), + UsagePercent: usage, + Load1: round2(load1), + Load5: round2(load5), + Load15: round2(load15), + }, + Memory: mem, + Disk: disk, + Network: netStats, + } + + return stats, nil +} + +func authMiddleware(token, requestSecret string, next http.Handler) http.Handler { + ns := &nonceState{seen: map[string]int64{}} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + auth := strings.TrimSpace(r.Header.Get("Authorization")) + xToken := strings.TrimSpace(r.Header.Get("X-Agent-Token")) + + ok := false + if strings.HasPrefix(strings.ToLower(auth), "bearer ") { + provided := strings.TrimSpace(auth[7:]) + if secureTokenEqual(provided, token) { + ok = true + } + } + if secureTokenEqual(xToken, token) { + ok = true + } + if !ok { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if !verifySignedRequest(r, requestSecret, ns) { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) +} + +func secureTokenEqual(provided, expected string) bool { + if len(provided) == 0 || len(expected) == 0 { + return false + } + return subtle.ConstantTimeCompare([]byte(provided), []byte(expected)) == 1 +} + +func verifySignedRequest(r *http.Request, secret string, ns *nonceState) bool { + if strings.TrimSpace(secret) == "" { + return true + } + tsRaw := strings.TrimSpace(r.Header.Get("X-Observer-Ts")) + nonce := strings.TrimSpace(r.Header.Get("X-Observer-Nonce")) + sigRaw := strings.TrimSpace(strings.ToLower(r.Header.Get("X-Observer-Signature"))) + if tsRaw == "" || nonce == "" || sigRaw == "" { + return false + } + ts, err := strconv.ParseInt(tsRaw, 10, 64) + if err != nil { + return false + } + now := time.Now().UTC().Unix() + if ts < now-signedRequestMaxSkewSeconds || ts > now+signedRequestMaxSkewSeconds { + return false + } + payload := r.Method + "\n" + r.URL.RequestURI() + "\n" + tsRaw + "\n" + nonce + mac := hmac.New(sha256.New, []byte(secret)) + _, _ = mac.Write([]byte(payload)) + wantHex := hex.EncodeToString(mac.Sum(nil)) + if subtle.ConstantTimeCompare([]byte(sigRaw), []byte(strings.ToLower(wantHex))) != 1 { + return false + } + + key := tsRaw + ":" + nonce + ns.lock() + defer ns.unlock() + if expiry, ok := ns.get(key); ok && expiry >= now { + return false + } + ns.set(key, now+signedRequestMaxSkewSeconds+60) + ns.prune(now) + return true +} + +func (n *nonceState) lock() { n.mu.Lock() } +func (n *nonceState) unlock() { n.mu.Unlock() } +func (n *nonceState) get(k string) (int64, bool) { + v, ok := n.seen[k] + return v, ok +} +func (n *nonceState) set(k string, exp int64) { + n.seen[k] = exp +} +func (n *nonceState) prune(now int64) { + if len(n.seen) < 4096 { + return + } + for k, exp := range n.seen { + if exp < now { + delete(n.seen, k) + } + } +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + _ = enc.Encode(v) +} + +func readUptimeSeconds() float64 { + if runtime.GOOS == "linux" { + data, err := os.ReadFile("/proc/uptime") + if err == nil { + parts := strings.Fields(string(data)) + if len(parts) > 0 { + if v, err := strconv.ParseFloat(parts[0], 64); err == nil { + return v + } + } + } + } + return 0 +} + +func readLoadAvg() (float64, float64, float64) { + if runtime.GOOS != "linux" { + return 0, 0, 0 + } + data, err := os.ReadFile("/proc/loadavg") + if err != nil { + return 0, 0, 0 + } + parts := strings.Fields(string(data)) + if len(parts) < 3 { + return 0, 0, 0 + } + l1, _ := strconv.ParseFloat(parts[0], 64) + l5, _ := strconv.ParseFloat(parts[1], 64) + l15, _ := strconv.ParseFloat(parts[2], 64) + return l1, l5, l15 +} + +func readCPUUsagePercent() float64 { + if runtime.GOOS != "linux" { + return 0 + } + + total, idle, err := readCPUCounters() + if err != nil { + return 0 + } + + cpuSampleMu.Lock() + defer cpuSampleMu.Unlock() + + if !prevCPUSet { + prevCPUTotal = total + prevCPUIdle = idle + prevCPUSet = true + return 0 + } + + var usage float64 + if total > prevCPUTotal { + dTotal := total - prevCPUTotal + dIdle := uint64(0) + if idle > prevCPUIdle { + dIdle = idle - prevCPUIdle + } + if dTotal > 0 { + if dIdle > dTotal { + dIdle = dTotal + } + usage = (float64(dTotal-dIdle) / float64(dTotal)) * 100 + } + } + + prevCPUTotal = total + prevCPUIdle = idle + + if usage < 0 { + usage = 0 + } + if usage > 100 { + usage = 100 + } + return round2(usage) +} + +func readCPUCounters() (uint64, uint64, error) { + data, err := os.ReadFile("/proc/stat") + if err != nil { + return 0, 0, err + } + + lines := strings.Split(string(data), "\n") + if len(lines) == 0 { + return 0, 0, errors.New("empty /proc/stat") + } + + fields := strings.Fields(lines[0]) + if len(fields) < 5 || fields[0] != "cpu" { + return 0, 0, errors.New("invalid cpu line in /proc/stat") + } + + var values []uint64 + for i := 1; i < len(fields); i++ { + v, convErr := strconv.ParseUint(fields[i], 10, 64) + if convErr != nil { + return 0, 0, convErr + } + values = append(values, v) + } + + total := uint64(0) + for _, v := range values { + total += v + } + idle := values[3] + if len(values) > 4 { + idle += values[4] // iowait + } + return total, idle, nil +} + +func readMemory() MemoryStats { + if runtime.GOOS != "linux" { + return MemoryStats{} + } + + f, err := os.Open("/proc/meminfo") + if err != nil { + return MemoryStats{} + } + defer f.Close() + + values := map[string]uint64{} + s := bufio.NewScanner(f) + for s.Scan() { + line := s.Text() + parts := strings.Split(line, ":") + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + fields := strings.Fields(strings.TrimSpace(parts[1])) + if len(fields) == 0 { + continue + } + v, err := strconv.ParseUint(fields[0], 10, 64) + if err != nil { + continue + } + values[key] = v * 1024 // kB -> bytes + } + + total := values["MemTotal"] + available := values["MemAvailable"] + if available == 0 { + available = values["MemFree"] + values["Buffers"] + values["Cached"] + } + used := uint64(0) + if total > available { + used = total - available + } + usedPercent := 0.0 + if total > 0 { + usedPercent = (float64(used) / float64(total)) * 100 + } + + swapTotal := values["SwapTotal"] + swapFree := values["SwapFree"] + swapUsed := uint64(0) + if swapTotal > swapFree { + swapUsed = swapTotal - swapFree + } + swapUsedPct := 0.0 + if swapTotal > 0 { + swapUsedPct = (float64(swapUsed) / float64(swapTotal)) * 100 + } + + return MemoryStats{ + TotalBytes: total, + AvailableBytes: available, + UsedBytes: used, + UsedPercent: round2(usedPercent), + SwapTotalBytes: swapTotal, + SwapFreeBytes: swapFree, + SwapUsedBytes: swapUsed, + SwapUsedPct: round2(swapUsedPct), + } +} + +type mountInfo struct { + Source string + MountPoint string + FSType string +} + +func readDiskStats() []DiskStats { + mounts := discoverMounts() + seen := make(map[string]struct{}, len(mounts)) + out := make([]DiskStats, 0, len(mounts)) + + for _, m := range mounts { + if _, ok := seen[m.MountPoint]; ok { + continue + } + seen[m.MountPoint] = struct{}{} + + total, free, err := statfsUsage(m.MountPoint) + if err != nil { + continue + } + used := uint64(0) + if total > free { + used = total - free + } + usedPercent := 0.0 + if total > 0 { + usedPercent = (float64(used) / float64(total)) * 100 + } + + device, readOnly, state, errorsCount, health, warnings := readDiskHealth(m.Source, m.FSType) + + out = append(out, DiskStats{ + Source: m.Source, + MountPoint: m.MountPoint, + FSType: m.FSType, + Device: device, + TotalBytes: total, + FreeBytes: free, + UsedBytes: used, + UsedPercent: round2(usedPercent), + ReadOnly: readOnly, + DeviceState: state, + ErrorsCount: errorsCount, + Health: health, + Warnings: warnings, + }) + } + + sort.Slice(out, func(i, j int) bool { return out[i].MountPoint < out[j].MountPoint }) + return out +} + +func discoverMounts() []mountInfo { + if runtime.GOOS != "linux" { + return []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}} + } + + f, err := os.Open("/proc/mounts") + if err != nil { + return []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}} + } + defer f.Close() + + skipFS := map[string]struct{}{ + "proc": {}, "sysfs": {}, "tmpfs": {}, "devtmpfs": {}, "devpts": {}, + "overlay": {}, "squashfs": {}, "cgroup": {}, "cgroup2": {}, "autofs": {}, + "securityfs": {}, "pstore": {}, "debugfs": {}, "tracefs": {}, "fusectl": {}, + } + + out := []mountInfo{} + s := bufio.NewScanner(f) + for s.Scan() { + line := s.Text() + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + fsType := fields[2] + if _, skip := skipFS[fsType]; skip { + continue + } + out = append(out, mountInfo{ + Source: fields[0], + MountPoint: fields[1], + FSType: fsType, + }) + } + if len(out) == 0 { + out = []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}} + } + return out +} + +func readDiskHealth(source, fsType string) (device string, readOnly bool, state string, errorsCount uint64, health string, warnings []string) { + health = "ok" + if !strings.HasPrefix(source, "/dev/") { + return "", false, "", 0, health, nil + } + + device = filepath.Base(source) + if strings.HasPrefix(device, "mapper/") { + device = strings.TrimPrefix(device, "mapper/") + } + device = strings.TrimPrefix(device, "/") + + roPath := filepath.Join("/sys/class/block", device, "ro") + if v, err := os.ReadFile(roPath); err == nil { + readOnly = strings.TrimSpace(string(v)) == "1" + if readOnly { + health = "warning" + warnings = append(warnings, "device is read-only") + } + } + + statePath := filepath.Join("/sys/class/block", device, "device", "state") + if v, err := os.ReadFile(statePath); err == nil { + state = strings.TrimSpace(string(v)) + if state != "" && state != "running" { + if health == "ok" { + health = "warning" + } + warnings = append(warnings, "device state is "+state) + } + } + + if fsType == "ext4" { + ext4ErrPath := filepath.Join("/sys/fs/ext4", device, "errors_count") + if v, err := os.ReadFile(ext4ErrPath); err == nil { + if parsed, convErr := strconv.ParseUint(strings.TrimSpace(string(v)), 10, 64); convErr == nil { + errorsCount = parsed + if errorsCount > 0 { + health = "critical" + warnings = append(warnings, fmt.Sprintf("ext4 errors_count=%d", errorsCount)) + } + } + } + } + + return device, readOnly, state, errorsCount, health, warnings +} + +func readNetworkStats() []NetworkStat { + if runtime.GOOS != "linux" { + return []NetworkStat{} + } + + f, err := os.Open("/proc/net/dev") + if err != nil { + return []NetworkStat{} + } + defer f.Close() + + out := []NetworkStat{} + s := bufio.NewScanner(f) + lineNo := 0 + for s.Scan() { + lineNo++ + if lineNo <= 2 { + continue + } + line := strings.TrimSpace(s.Text()) + parts := strings.Split(line, ":") + if len(parts) != 2 { + continue + } + iface := strings.TrimSpace(parts[0]) + fields := strings.Fields(parts[1]) + if len(fields) < 16 { + continue + } + rx, err1 := strconv.ParseUint(fields[0], 10, 64) + rxPackets, errP1 := strconv.ParseUint(fields[1], 10, 64) + rxDrops, errD1 := strconv.ParseUint(fields[3], 10, 64) + tx, err2 := strconv.ParseUint(fields[8], 10, 64) + txPackets, errP2 := strconv.ParseUint(fields[9], 10, 64) + txDrops, errD2 := strconv.ParseUint(fields[11], 10, 64) + if err1 != nil || err2 != nil || errP1 != nil || errP2 != nil || errD1 != nil || errD2 != nil { + continue + } + out = append(out, NetworkStat{ + Interface: iface, + RxBytes: rx, + TxBytes: tx, + RxPackets: rxPackets, + TxPackets: txPackets, + RxDrops: rxDrops, + TxDrops: txDrops, + }) + } + + sort.Slice(out, func(i, j int) bool { return out[i].Interface < out[j].Interface }) + return out +} + +func round2(v float64) float64 { + return math.Round(v*100) / 100 +} + +// DefaultConfigPath returns default location used by agent process. +func DefaultConfigPath() string { + return filepath.Join(".", "agent.json") +} diff --git a/internal/agent/statfs_unix.go b/internal/agent/statfs_unix.go new file mode 100644 index 0000000..bb7df79 --- /dev/null +++ b/internal/agent/statfs_unix.go @@ -0,0 +1,13 @@ +//go:build !windows + +package agent + +import "syscall" + +func statfsUsage(path string) (total, free uint64, err error) { + var fs syscall.Statfs_t + if err := syscall.Statfs(path, &fs); err != nil { + return 0, 0, err + } + return fs.Blocks * uint64(fs.Bsize), fs.Bavail * uint64(fs.Bsize), nil +} diff --git a/internal/agent/statfs_windows.go b/internal/agent/statfs_windows.go new file mode 100644 index 0000000..e973fa6 --- /dev/null +++ b/internal/agent/statfs_windows.go @@ -0,0 +1,10 @@ +//go:build windows + +package agent + +// statfsUsage is a compatibility fallback for Windows builds where +// syscall.Statfs is unavailable. We return zero-sized stats instead of +// failing compilation; disk health/usage can be extended with native APIs. +func statfsUsage(path string) (total, free uint64, err error) { + return 0, 0, nil +} diff --git a/internal/agent/top.go b/internal/agent/top.go new file mode 100644 index 0000000..ead5b9e --- /dev/null +++ b/internal/agent/top.go @@ -0,0 +1,431 @@ +package agent + +import ( + "bufio" + "context" + "errors" + "io/fs" + "os" + "os/user" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// ProcessStat is one process summary for /api/v1/top. +type ProcessStat struct { + PID int `json:"pid"` + User string `json:"user"` + Command string `json:"command"` + CPUPercent float64 `json:"cpu_percent"` + RSSBytes uint64 `json:"rss_bytes"` + VSZBytes uint64 `json:"vsz_bytes"` + IOReadTot uint64 `json:"io_read_bytes"` + IOWriteTot uint64 `json:"io_write_bytes"` +} + +// TopResponse is the payload returned by /api/v1/top. +type TopResponse struct { + Timestamp time.Time `json:"timestamp"` + SampleMs int64 `json:"sample_ms"` + TotalProcs int `json:"total_procs"` + TopByCPU []ProcessStat `json:"top_by_cpu"` + TopByMemory []ProcessStat `json:"top_by_memory"` + TopByIO []ProcessStat `json:"top_by_io"` +} + +// CollectTopProcesses takes two samples of /proc/[pid]/stat separated by +// sampleWindow to compute CPU%, then returns the top N processes by CPU, by +// RSS, and by IO total. Linux-only; other OSes return an empty response. +func CollectTopProcesses(sampleWindow time.Duration, limit int) (TopResponse, error) { + if runtime.GOOS != "linux" { + return TopResponse{Timestamp: time.Now().UTC()}, nil + } + if sampleWindow <= 0 { + sampleWindow = 250 * time.Millisecond + } + if sampleWindow > 2*time.Second { + sampleWindow = 2 * time.Second + } + if limit <= 0 || limit > 200 { + limit = 20 + } + + hz := clockTicksPerSecond() + pageSize := uint64(os.Getpagesize()) + + first, err := snapshotProcesses() + if err != nil { + return TopResponse{}, err + } + time.Sleep(sampleWindow) + second, err := snapshotProcesses() + if err != nil { + return TopResponse{}, err + } + + usernameCache := newUsernameCache() + elapsedTicks := float64(sampleWindow.Seconds()) * hz + if elapsedTicks <= 0 { + elapsedTicks = 1 + } + + merged := make([]ProcessStat, 0, len(second)) + for pid, s2 := range second { + s1, ok := first[pid] + cpu := 0.0 + if ok { + dTicks := float64((s2.utime + s2.stime) - (s1.utime + s1.stime)) + if dTicks > 0 { + cpu = (dTicks / elapsedTicks) * 100.0 + } + } + if cpu < 0 { + cpu = 0 + } + + merged = append(merged, ProcessStat{ + PID: pid, + User: usernameCache.lookup(s2.uid), + Command: s2.command, + CPUPercent: round2(cpu), + RSSBytes: s2.rssPages * pageSize, + VSZBytes: s2.vsize, + IOReadTot: s2.ioRead, + IOWriteTot: s2.ioWrite, + }) + } + + byCPU := topN(merged, limit, func(a, b ProcessStat) bool { return a.CPUPercent > b.CPUPercent }) + byMem := topN(merged, limit, func(a, b ProcessStat) bool { return a.RSSBytes > b.RSSBytes }) + byIO := topN(merged, limit, func(a, b ProcessStat) bool { + return (a.IOReadTot + a.IOWriteTot) > (b.IOReadTot + b.IOWriteTot) + }) + + return TopResponse{ + Timestamp: time.Now().UTC(), + SampleMs: sampleWindow.Milliseconds(), + TotalProcs: len(merged), + TopByCPU: byCPU, + TopByMemory: byMem, + TopByIO: byIO, + }, nil +} + +type procSample struct { + pid int + command string + utime uint64 + stime uint64 + vsize uint64 + rssPages uint64 + uid int + ioRead uint64 + ioWrite uint64 +} + +func snapshotProcesses() (map[int]procSample, error) { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil, err + } + out := make(map[int]procSample, 256) + for _, e := range entries { + if !e.IsDir() { + continue + } + pid, err := strconv.Atoi(e.Name()) + if err != nil || pid <= 0 { + continue + } + s, ok := readProcSample(pid) + if !ok { + continue + } + out[pid] = s + } + return out, nil +} + +func readProcSample(pid int) (procSample, bool) { + pidStr := strconv.Itoa(pid) + statData, err := os.ReadFile("/proc/" + pidStr + "/stat") + if err != nil { + return procSample{}, false + } + + // comm is in parens and may contain spaces; parse after the last ')'. + line := string(statData) + rp := strings.LastIndexByte(line, ')') + if rp <= 0 { + return procSample{}, false + } + lp := strings.IndexByte(line, '(') + if lp < 0 || lp >= rp { + return procSample{}, false + } + comm := line[lp+1 : rp] + rest := strings.Fields(line[rp+2:]) + // After comm and the state char, indices inside `rest`: + // 0: state ... but we split from after ') '. fields[0]=state, fields[1]=ppid, + // fields[2]=pgrp, ..., fields[11]=utime, fields[12]=stime, fields[19]=vsize, fields[20]=rss. + if len(rest) < 22 { + return procSample{}, false + } + utime, _ := strconv.ParseUint(rest[11], 10, 64) + stime, _ := strconv.ParseUint(rest[12], 10, 64) + vsize, _ := strconv.ParseUint(rest[20], 10, 64) + rssPages, _ := strconv.ParseUint(rest[21], 10, 64) + + uid := readProcUID(pidStr) + ioRead, ioWrite := readProcIO(pidStr) + + return procSample{ + pid: pid, + command: comm, + utime: utime, + stime: stime, + vsize: vsize, + rssPages: rssPages, + uid: uid, + ioRead: ioRead, + ioWrite: ioWrite, + }, true +} + +func readProcUID(pidStr string) int { + data, err := os.ReadFile("/proc/" + pidStr + "/status") + if err != nil { + return -1 + } + for _, line := range strings.Split(string(data), "\n") { + if strings.HasPrefix(line, "Uid:") { + fields := strings.Fields(line) + if len(fields) >= 2 { + if v, err := strconv.Atoi(fields[1]); err == nil { + return v + } + } + return -1 + } + } + return -1 +} + +func readProcIO(pidStr string) (uint64, uint64) { + data, err := os.ReadFile("/proc/" + pidStr + "/io") + if err != nil { + return 0, 0 + } + var rb, wb uint64 + for _, line := range strings.Split(string(data), "\n") { + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + val := strings.TrimSpace(parts[1]) + v, _ := strconv.ParseUint(val, 10, 64) + switch key { + case "read_bytes": + rb = v + case "write_bytes": + wb = v + } + } + return rb, wb +} + +var cachedClockTicks float64 +var cachedClockOnce sync.Once + +func clockTicksPerSecond() float64 { + cachedClockOnce.Do(func() { + // SC_CLK_TCK is almost always 100 on Linux; read from /proc/self/stat + // + uptime as a sanity check would be nicer but we accept the default. + cachedClockTicks = 100.0 + }) + return cachedClockTicks +} + +type usernameCache struct { + cache map[int]string +} + +func newUsernameCache() *usernameCache { + return &usernameCache{cache: make(map[int]string, 16)} +} + +func (c *usernameCache) lookup(uid int) string { + if uid < 0 { + return "-" + } + if v, ok := c.cache[uid]; ok { + return v + } + u, err := user.LookupId(strconv.Itoa(uid)) + if err != nil || u == nil { + name := strconv.Itoa(uid) + c.cache[uid] = name + return name + } + c.cache[uid] = u.Username + return u.Username +} + +func topN(in []ProcessStat, n int, less func(a, b ProcessStat) bool) []ProcessStat { + cp := make([]ProcessStat, len(in)) + copy(cp, in) + sort.Slice(cp, func(i, j int) bool { return less(cp[i], cp[j]) }) + if len(cp) > n { + cp = cp[:n] + } + return cp +} + +// DirStat is one directory entry in a du-style listing. +type DirStat struct { + Path string `json:"path"` + Name string `json:"name"` + Bytes uint64 `json:"bytes"` + Files uint64 `json:"files"` +} + +// DUResponse is returned by /api/v1/du. +type DUResponse struct { + Timestamp time.Time `json:"timestamp"` + Root string `json:"root"` + Entries []DirStat `json:"entries"` + Truncated bool `json:"truncated"` +} + +// CollectDirSizes lists immediate children of root (only directories) and +// sums file sizes under each child recursively, respecting ctx deadline. +// Returned entries are sorted by Bytes descending. Truncated is true if the +// walker stopped early due to timeout. +func CollectDirSizes(ctx context.Context, root string, limit int) (DUResponse, error) { + root = strings.TrimSpace(root) + if root == "" { + root = "/" + } + absRoot, err := filepath.Abs(root) + if err != nil { + return DUResponse{}, err + } + info, err := os.Stat(absRoot) + if err != nil { + return DUResponse{}, err + } + if !info.IsDir() { + return DUResponse{}, errors.New("root is not a directory") + } + if limit <= 0 || limit > 100 { + limit = 15 + } + + entries, err := os.ReadDir(absRoot) + if err != nil { + return DUResponse{}, err + } + + truncated := false + out := make([]DirStat, 0, len(entries)) + for _, e := range entries { + if ctx.Err() != nil { + truncated = true + break + } + if !e.IsDir() { + continue + } + name := e.Name() + // Skip virtual filesystems when rooted at /. + if absRoot == "/" && isSkippedSystemDir(name) { + continue + } + full := filepath.Join(absRoot, name) + size, files, stopped := sumDirectory(ctx, full) + if stopped { + truncated = true + } + out = append(out, DirStat{ + Path: full, + Name: name, + Bytes: size, + Files: files, + }) + if stopped { + break + } + } + + sort.Slice(out, func(i, j int) bool { return out[i].Bytes > out[j].Bytes }) + if len(out) > limit { + out = out[:limit] + } + + return DUResponse{ + Timestamp: time.Now().UTC(), + Root: absRoot, + Entries: out, + Truncated: truncated, + }, nil +} + +func isSkippedSystemDir(name string) bool { + switch name { + case "proc", "sys", "dev", "run", "tmp": + return true + } + return false +} + +func sumDirectory(ctx context.Context, path string) (uint64, uint64, bool) { + var total uint64 + var files uint64 + stopped := false + + walkFn := func(p string, d fs.DirEntry, err error) error { + if err != nil { + if d != nil && d.IsDir() { + return filepath.SkipDir + } + return nil + } + if ctx.Err() != nil { + stopped = true + return filepath.SkipAll + } + if d.IsDir() { + // Skip known pseudo filesystems we may cross into. + name := d.Name() + if p != path && (name == "proc" || name == "sys" || name == "dev") { + return filepath.SkipDir + } + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + total += uint64(info.Size()) + files++ + return nil + } + _ = filepath.WalkDir(path, walkFn) + return total, files, stopped +} + +// scanLinesToFields is a tiny helper for tests and future use. +func scanLinesToFields(data []byte) [][]string { + var out [][]string + sc := bufio.NewScanner(strings.NewReader(string(data))) + for sc.Scan() { + out = append(out, strings.Fields(sc.Text())) + } + return out +} diff --git a/internal/cli/app.go b/internal/cli/app.go new file mode 100644 index 0000000..f98da9b --- /dev/null +++ b/internal/cli/app.go @@ -0,0 +1,2969 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "text/tabwriter" + "time" + + "golang.org/x/term" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +type App struct { + out io.Writer + err io.Writer +} + +type multiFlag []string + +func (m *multiFlag) String() string { + if m == nil { + return "" + } + return strings.Join(*m, ",") +} + +func (m *multiFlag) Set(v string) error { + *m = append(*m, v) + return nil +} + +func New(out, err io.Writer) *App { + return &App{out: out, err: err} +} + +func (a *App) Run(args []string) int { + root := flag.NewFlagSet("pxmon", flag.ContinueOnError) + root.SetOutput(a.err) + root.Usage = func() { + a.printRootHelp() + } + + configPath := root.String("config", "", "Path to encrypted cluster registry file") + jsonOut := root.Bool("json", false, "Output as JSON") + + if err := root.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + + return a.runRootCommand(root.Args(), *configPath, *jsonOut, true) +} + +func (a *App) runRootCommand(rest []string, configPath string, jsonOut bool, allowShell bool) int { + if len(rest) == 0 { + if allowShell && term.IsTerminal(int(os.Stdin.Fd())) && term.IsTerminal(int(os.Stdout.Fd())) { + return a.runTUI(nil, configPath, jsonOut) + } + a.printRootHelp() + return 0 + } + + switch rest[0] { + case "help", "--help", "-h": + a.printRootHelp() + return 0 + case "locker": + return a.runLocker(rest[1:], configPath, jsonOut) + case "cluster": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runCluster(rest[1:], configPath, jsonOut) + case "shell": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runTUI(rest[1:], configPath, jsonOut) + case "tui": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runTUI(rest[1:], configPath, jsonOut) + case "clusters": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runClusters(rest[1:], configPath, jsonOut) + case "network": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runNetwork(rest[1:], configPath, jsonOut) + case "bot": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runBot(rest[1:], configPath, jsonOut) + case "config": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runConfig(rest[1:], configPath) + case "export": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runConfigExport(rest[1:], configPath) + case "import": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runConfigImport(rest[1:], configPath) + case "explain": + if code, ok := a.guardLocker(configPath, rest[0]); ok { + return code + } + return a.runExplain(rest[1:]) + default: + fmt.Fprintf(a.err, "unknown command %q\n\n", rest[0]) + a.printRootHelp() + return 2 + } +} + +func (a *App) guardLocker(configPath, command string) (int, bool) { + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1, true + } + svc := cluster.NewService(store) + locked, _, err := svc.IsLocked() + if err != nil { + fmt.Fprintf(a.err, "locker check failed: %v\n", err) + return 1, true + } + if !locked { + return 0, false + } + svc.AuditLocker("locker_blocked_command", command) + fmt.Fprintf(a.err, "locker is enabled: command %q is blocked until unlock\n", command) + fmt.Fprintln(a.err, "Run: pxmon locker unlock") + return 1, true +} + +func (a *App) runTUI(args []string, configPath string, jsonOut bool) int { + return a.runDashboard("tui", args, configPath, jsonOut, "overview") +} + +func (a *App) runNetwork(args []string, configPath string, jsonOut bool) int { + return a.runDashboard("network", args, configPath, jsonOut, "network") +} + +func (a *App) runClusters(args []string, configPath string, jsonOut bool) int { + return a.runDashboard("clusters", args, configPath, jsonOut, "clusters") +} + +func (a *App) runDashboard(name string, args []string, configPath string, jsonOut bool, view string) int { + if jsonOut { + fmt.Fprintln(a.err, "--json is not supported for interactive TUI mode") + return 2 + } + + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(a.err) + interval := fs.Duration("interval", 2*time.Second, "Refresh interval") + iface := fs.String("iface", "", "Initial network interface") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 1 { + fmt.Fprintf(a.err, "usage: pxmon %s [name-or-id]\n", name) + return 2 + } + + selector := "" + if fs.NArg() == 1 { + selector = fs.Arg(0) + } + + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + + policy, err := svc.GetAlertPolicy(selector) + if err != nil { + if errors.Is(err, cluster.ErrNoActiveCluster) && strings.TrimSpace(selector) == "" { + policy = cluster.AlertPolicy{ + CPUWarnPercent: 85, + RAMWarnPercent: 90, + SwapWarnPercent: 80, + DiskWarnPercent: 90, + NetWarnMbps: 300, + } + } else { + fmt.Fprintf(a.err, "load alert policy: %v\n", err) + return 1 + } + } + + return a.runClusterMonitor(svc, selector, MonitorOptions{ + Interval: *interval, + InitialIface: strings.TrimSpace(*iface), + AlertPolicy: policy, + InitialView: view, + }) +} + +func (a *App) runCluster(args []string, configPath string, jsonOut bool) int { + if len(args) == 0 { + a.printClusterHelp() + return 0 + } + + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + svc.AttachNetworkStore(history.NewNetworkStore(svc.DataDir())) + + sub := args[0] + rest := args[1:] + + switch sub { + case "connect", "add": + return a.runClusterConnect(svc, rest, jsonOut) + case "list", "ls": + return a.runClusterList(svc, jsonOut) + case "show", "get": + return a.runClusterShow(svc, rest, jsonOut) + case "current": + return a.runClusterCurrent(svc, jsonOut) + case "use": + return a.runClusterUse(svc, rest) + case "disconnect", "remove", "rm": + return a.runClusterDisconnect(svc, rest) + case "set-auth", "auth", "password", "passwd": + return a.runClusterSetAuth(svc, rest) + case "openssh", "ssh": + return a.runClusterOpenSSH(svc, rest) + case "exec", "run": + return a.runClusterExec(svc, rest, jsonOut) + case "ping", "check": + return a.runClusterPing(svc, rest, jsonOut) + case "bootstrap": + return a.runClusterBootstrap(svc, rest, jsonOut) + case "agent": + return a.runClusterAgent(svc, rest, jsonOut) + case "stats": + return a.runClusterStats(svc, rest, jsonOut) + case "usage": + return a.runClusterUsage(svc, rest, jsonOut) + case "slo", "availability": + return a.runClusterSLO(svc, rest, jsonOut) + case "traffic": + return a.runClusterTraffic(svc, rest, jsonOut) + case "graph": + return a.runClusterGraph(svc, rest, jsonOut) + case "p95": + return a.runClusterP95(svc, rest, jsonOut) + case "alert", "alerts": + return a.runClusterAlert(svc, rest, jsonOut) + case "alert-routing", "routing": + return a.runClusterAlertRouting(svc, rest, jsonOut) + case "alert-vm", "vm-alert": + return a.runClusterVMAlert(svc, rest, jsonOut) + case "software", "plugins": + return a.runClusterSoftware(svc, rest, jsonOut) + case "tag", "tags": + return a.runClusterTag(svc, rest, jsonOut) + case "kvm-tag", "vm-tag": + return a.runClusterKVMTag(svc, rest, jsonOut) + case "change-history", "changes": + return a.runClusterChangeHistory(svc, rest, jsonOut) + case "drift": + return a.runClusterDrift(svc, rest, jsonOut) + case "runbook": + return a.runClusterRunbook(svc, rest, jsonOut) + case "runbook-trigger": + return a.runClusterRunbookTrigger(svc, rest, jsonOut) + case "schedule", "scheduler": + return a.runClusterSchedule(svc, rest, jsonOut) + case "report": + return a.runClusterReport(svc, rest, jsonOut) + case "backup": + return a.runClusterBackup(svc, rest, jsonOut) + case "repo-tunnel", "repo", "repo-tunneling": + return a.runClusterRepoTunnel(svc, rest, jsonOut) + case "capacity": + return a.runClusterCapacity(svc, rest, jsonOut) + case "help", "--help", "-h": + a.printClusterHelp() + return 0 + default: + fmt.Fprintf(a.err, "unknown cluster command %q\n\n", sub) + a.printClusterHelp() + return 2 + } +} + +func (a *App) runClusterConnect(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster connect", flag.ContinueOnError) + fs.SetOutput(a.err) + + name := fs.String("name", "", "Cluster name") + host := fs.String("host", "", "SSH host or IP") + port := fs.Int("port", 22, "SSH port") + user := fs.String("user", "", "SSH user") + transport := fs.String("type", "direct", "Network transport to the agent: direct|ipfabric (ipfabric tunnels HTTP over the SSH connection)") + auth := fs.String("auth", "key", "SSH auth method: key|password") + password := fs.String("password", "", "SSH password") + storePassword := fs.Bool("store-password", false, "Persist password in encrypted local store") + keyPath := fs.String("key-path", "", "SSH private key path (for auth=key)") + keyPassphrase := fs.String("key-passphrase", "", "SSH private key passphrase") + keyPassphraseFile := fs.String("key-passphrase-file", "", "File containing SSH private key passphrase") + storeKeyPass := fs.Bool("store-key-passphrase", false, "Persist key passphrase in encrypted local store") + storeKeyPassFile := fs.Bool("store-key-passphrase-file", false, "Persist key passphrase file path in encrypted local store") + insecureHostKey := fs.Bool("insecure-host-key", false, "Disable SSH host-key verification") + skipCheck := fs.Bool("skip-check", false, "Skip SSH connectivity check during connect") + allowUnreachable := fs.Bool("allow-unreachable", false, "Save even if SSH check fails") + force := fs.Bool("force", false, "Overwrite existing cluster with same name") + + if err := fs.Parse(args); err != nil { + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(a.err, "unexpected argument(s): %s\n", strings.Join(fs.Args(), " ")) + return 2 + } + resolvedKeyPassphrase, err := readSecretValueFileFlag("key passphrase", *keyPassphrase, *keyPassphraseFile) + if err != nil { + fmt.Fprintf(a.err, "connect cluster: %v\n", err) + return 2 + } + + authMethod := cluster.AuthMethod(strings.ToLower(strings.TrimSpace(*auth))) + transportMode, tErr := parseTransportFlag(*transport) + if tErr != nil { + fmt.Fprintf(a.err, "connect cluster: %v\n", tErr) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + c, probe, err := svc.Connect(ctx, cluster.ConnectOptions{ + Name: *name, + Host: *host, + Port: *port, + User: *user, + Transport: transportMode, + AuthMethod: authMethod, + Password: *password, + StorePassword: *storePassword, + KeyPath: *keyPath, + KeyPassphrase: resolvedKeyPassphrase, + KeyPassphraseFile: *keyPassphraseFile, + StoreKeyPassphrase: *storeKeyPass, + StoreKeyPassphraseFile: *storeKeyPassFile, + InsecureHostKey: *insecureHostKey, + SkipCheck: *skipCheck, + AllowUnreachable: *allowUnreachable, + Force: *force, + }) + if err != nil { + fmt.Fprintf(a.err, "connect cluster: %v\n", err) + return 1 + } + + if jsonOut { + c = sanitizeCluster(c, false) + _ = writeJSON(a.out, map[string]any{ + "cluster": c, + "probe": probe, + }) + return 0 + } + + fmt.Fprintf(a.out, "Connected %q -> %s@%s:%d\n", c.Name, c.User, c.Host, c.Port) + if c.InsecureHostKey { + fmt.Fprintln(a.err, "warning: insecure host-key mode is ON (MITM risk). Use only in trusted/private networks.") + } + if !probe.CheckedAt.IsZero() { + if probe.Reachable { + fmt.Fprintf(a.out, "SSH probe: OK (%dms)\n", probe.LatencyMS) + } else { + fmt.Fprintf(a.out, "SSH probe: FAILED %s\n", probe.Error) + } + } + fmt.Fprintf(a.out, "Active cluster: %s\n", c.Name) + return 0 +} + +func (a *App) runClusterList(svc *cluster.Service, jsonOut bool) int { + clusters, activeID, err := svc.List() + if err != nil { + fmt.Fprintf(a.err, "list clusters: %v\n", err) + return 1 + } + + if jsonOut { + items := make([]map[string]any, 0, len(clusters)) + for _, c := range clusters { + items = append(items, map[string]any{ + "cluster": sanitizeCluster(c, false), + "repo_tunnel": c.RepoTunnel, + }) + } + _ = writeJSON(a.out, map[string]any{ + "active_cluster_id": activeID, + "clusters": items, + }) + return 0 + } + + if len(clusters) == 0 { + fmt.Fprintln(a.out, "No clusters connected yet.") + return 0 + } + + expectedVersion := svc.ExpectedAgentVersion() + type probeResult struct { + reachable bool + sshReachable bool + agentReachable bool + versionMismatch bool + version string + } + probes := make([]probeResult, len(clusters)) + var wg sync.WaitGroup + for i := range clusters { + wg.Add(1) + go func(i int) { + defer wg.Done() + c := clusters[i] + res := probeResult{version: strings.TrimSpace(c.Agent.Version)} + // Node reachability is based on SSH/TCP socket availability and is + // independent from agent health/version. + addr := net.JoinHostPort(c.Host, fmt.Sprintf("%d", c.Port)) + conn, dialErr := net.DialTimeout("tcp", addr, 2*time.Second) + if dialErr == nil { + res.sshReachable = true + res.reachable = true + _ = conn.Close() + } + if c.Agent.Installed { + ctx, cancel := context.WithTimeout(context.Background(), 2500*time.Millisecond) + ping, pingErr := svc.PingAgent(ctx, c.ID) + cancel() + if pingErr == nil && ping.Reachable && ping.StatusCode < 400 { + res.agentReachable = true + if strings.TrimSpace(ping.Version) != "" { + res.version = strings.TrimSpace(ping.Version) + } + } + if strings.TrimSpace(expectedVersion) != "" && strings.TrimSpace(res.version) != "" && res.version != expectedVersion { + res.versionMismatch = true + } + } + probes[i] = res + }(i) + } + wg.Wait() + + type listRow struct { + state string + stateColor string + name string + target string + auth string + agentPlain string + agentColor string + software string + softwareColor string + repoGW string + repoGWColor string + tags string + updated string + id string + } + rows := make([]listRow, 0, len(clusters)) + versionWarnings := make([]string, 0, len(clusters)) + for i, c := range clusters { + probe := probes[i] + isActive := c.ID == activeID + + state := "DOWN" + stateColor := "crit" + if probe.reachable { + state = "UP" + stateColor = "ok" + } + + auth := string(c.AuthMethod) + if c.InsecureHostKey { + auth += "(insecure-host-key)" + } + if c.Transport == cluster.TransportIPFabric { + auth += "(ipfabric)" + } + + agentStatus := "none" + agentColor := "warn" + if c.Agent.Installed { + version := probe.version + if version == "" { + version = "unknown" + } + agentState := "down" + if probe.agentReachable { + agentState = "up" + } + if probe.versionMismatch { + agentState = "mismatch" + } + agentStatus = fmt.Sprintf("%s v%s", agentState, shortVersion(version)) + if probe.versionMismatch { + agentStatus += "!=" + shortVersion(expectedVersion) + } + switch { + case probe.versionMismatch: + agentColor = "warn" + case probe.agentReachable: + agentColor = "ok" + default: + agentColor = "crit" + } + } + + software := c.Software.Summary() + softwareColor := "ok" + switch software { + case "-", "": + software = "-" + softwareColor = "crit" + case "none": + softwareColor = "warn" + } + repoGW := "-" + repoGWColor := "dim" + if c.RepoTunnel.Enabled { + repoGW = repoTunnelDisplay(c.RepoTunnel.Proxy) + repoGWColor = "ok" + } + + rows = append(rows, listRow{ + state: state, + stateColor: stateColor, + name: ternary(isActive, "*"+c.Name, c.Name), + target: fmt.Sprintf("%s@%s:%d", c.User, c.Host, c.Port), + auth: auth, + agentPlain: agentStatus, + agentColor: agentColor, + software: software, + softwareColor: softwareColor, + repoGW: repoGW, + repoGWColor: repoGWColor, + tags: ternary(len(c.Tags) == 0, "-", strings.Join(c.Tags, ",")), + updated: c.UpdatedAt.Local().Format("2006-01-02 15:04:05"), + id: c.ID, + }) + + if c.Agent.Installed && strings.TrimSpace(probe.version) != "" { + if ok, latest, known := svc.CompareAgentVersion(probe.version); known && !ok { + versionWarnings = append(versionWarnings, + fmt.Sprintf("%s: agent %s is outdated (latest known: %s), some features may not work", + c.Name, probe.version, latest)) + } else if probe.versionMismatch { + versionWarnings = append(versionWarnings, + fmt.Sprintf("%s: agent %s differs from local %s, some features may not work", + c.Name, probe.version, expectedVersion)) + } + } + } + + wState := maxLen("STATE", func(r listRow) string { return r.state }, rows) + wName := maxLen("NAME", func(r listRow) string { return r.name }, rows) + wTarget := maxLen("TARGET", func(r listRow) string { return r.target }, rows) + wAuth := maxLen("AUTH", func(r listRow) string { return r.auth }, rows) + wAgent := maxLen("AGENT", func(r listRow) string { return r.agentPlain }, rows) + wSoftware := maxLen("SOFTWARE", func(r listRow) string { return r.software }, rows) + wRepoGW := maxLen("REPO-GW", func(r listRow) string { return r.repoGW }, rows) + wTags := maxLen("TAGS", func(r listRow) string { return r.tags }, rows) + wUpdated := maxLen("UPDATED", func(r listRow) string { return r.updated }, rows) + + wState = clamp(wState, 5, 6) + wName = clamp(wName, 6, 16) + wTarget = clamp(wTarget, 12, 28) + wAuth = clamp(wAuth, 3, 22) + wAgent = clamp(wAgent, 8, 32) + wSoftware = clamp(wSoftware, 4, 14) + wRepoGW = clamp(wRepoGW, 7, 22) + wTags = clamp(wTags, 4, 16) + wUpdated = clamp(wUpdated, 19, 19) + + if term.IsTerminal(int(os.Stdout.Fd())) { + if width, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil && width > 0 { + sep := 2 * 8 + for { + total := wState + wName + wTarget + wAuth + wAgent + wSoftware + wRepoGW + wTags + wUpdated + sep + if total <= width { + break + } + shrunk := false + shrunk = shrinkOne(&wAgent, 8) || shrunk + shrunk = shrinkOne(&wTarget, 12) || shrunk + shrunk = shrinkOne(&wName, 6) || shrunk + shrunk = shrinkOne(&wAuth, 3) || shrunk + shrunk = shrinkOne(&wRepoGW, 7) || shrunk + shrunk = shrinkOne(&wTags, 4) || shrunk + shrunk = shrinkOne(&wSoftware, 4) || shrunk + shrunk = shrinkOne(&wUpdated, 10) || shrunk + if !shrunk { + break + } + } + } + } + + header := strings.Join([]string{ + colorizeCell("STATE", wState, "header"), + colorizeCell("NAME", wName, "header"), + colorizeCell("TARGET", wTarget, "header"), + colorizeCell("AUTH", wAuth, "header"), + colorizeCell("AGENT", wAgent, "header"), + colorizeCell("SOFTWARE", wSoftware, "header"), + colorizeCell("REPO-GW", wRepoGW, "header"), + colorizeCell("TAGS", wTags, "header"), + colorizeCell("UPDATED", wUpdated, "header"), + colorHeader("ID"), + }, " ") + fmt.Fprintln(a.out, header) + for _, r := range rows { + nameColor := "value" + if strings.HasPrefix(r.name, "*") { + nameColor = "accent" + } + line := strings.Join([]string{ + colorizeCell(r.state, wState, r.stateColor), + colorizeCell(r.name, wName, nameColor), + colorizeCell(r.target, wTarget, "blue"), + colorizeCell(r.auth, wAuth, "magenta"), + colorizeCell(r.agentPlain, wAgent, r.agentColor), + colorizeCell(r.software, wSoftware, r.softwareColor), + colorizeCell(r.repoGW, wRepoGW, r.repoGWColor), + colorizeCell(r.tags, wTags, "dim"), + colorizeCell(r.updated, wUpdated, "dim"), + colorMuted(r.id), + }, " ") + fmt.Fprintln(a.out, line) + } + if len(versionWarnings) > 0 { + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, colorWarn("Warnings:")) + for _, w := range versionWarnings { + fmt.Fprintf(a.out, " %s %s\n", colorWarn("!"), colorWarn(w)) + } + } + + return 0 +} + +func (a *App) runClusterShow(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster show", flag.ContinueOnError) + fs.SetOutput(a.err) + if err := fs.Parse(args); err != nil { + return 2 + } + + selector := "" + if fs.NArg() > 0 { + selector = fs.Arg(0) + } + + c, err := svc.Get(selector) + if err != nil { + if errors.Is(err, cluster.ErrNoActiveCluster) { + fmt.Fprintln(a.err, "no active cluster") + return 1 + } + fmt.Fprintf(a.err, "show cluster: %v\n", err) + return 1 + } + + c = sanitizeCluster(c, false) + if jsonOut { + _ = writeJSON(a.out, c) + return 0 + } + + hostKey := ternary(c.InsecureHostKey, "insecure", "strict") + hostKeyColored := colorOK(hostKey) + if c.InsecureHostKey { + hostKeyColored = colorWarn(hostKey) + } + agentStatusText := ternary(c.Agent.Installed, "installed", "not installed") + agentStatusColored := colorWarn(agentStatusText) + if c.Agent.Installed { + agentStatusColored = colorOK(agentStatusText) + } + + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Name: "), colorAccent(c.Name)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("ID: "), colorMuted(c.ID)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Host: "), colorBlue(c.Host)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Port: "), colorBlue(fmt.Sprintf("%d", c.Port))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("User: "), colorValue(c.User)) + transportText := string(c.Transport) + if strings.TrimSpace(transportText) == "" { + transportText = string(cluster.TransportDirect) + } + transportColored := colorInfo(transportText) + if c.Transport == cluster.TransportIPFabric { + transportColored = colorWarn(transportText + " (agent tunneled via SSH)") + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Transport: "), transportColored) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Auth: "), colorMagenta(string(c.AuthMethod))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Password: "), colorDim(printableSecret(c.Password))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Key path: "), colorInfo(emptyFallback(c.KeyPath, "(empty)"))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Key phrase: "), colorDim(printableSecret(c.KeyPassphrase))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Phrase file:"), colorInfo(emptyFallback(c.KeyPassphraseFile, "(empty)"))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Host key: "), hostKeyColored) + fmt.Fprintf(a.out, "%s %s / %s / %s / %s / %s\n", + colorLabel("Alerts: "), + colorWarn(fmt.Sprintf("CPU %.1f%%", c.Alerts.CPUWarnPercent)), + colorWarn(fmt.Sprintf("RAM %.1f%%", c.Alerts.RAMWarnPercent)), + colorWarn(fmt.Sprintf("Swap %.1f%%", c.Alerts.SwapWarnPercent)), + colorWarn(fmt.Sprintf("Disk %.1f%%", c.Alerts.DiskWarnPercent)), + colorWarn(fmt.Sprintf("Net %.1f Mbps", c.Alerts.NetWarnMbps)), + ) + fmt.Fprintf(a.out, "%s %s / %s / %s\n", + colorLabel("VM alerts: "), + colorWarn(fmt.Sprintf("enabled=%t", c.VMAlerts.Enabled)), + colorWarn(fmt.Sprintf("warn_on_shutoff=%t", c.VMAlerts.WarnOnShutoff)), + colorWarn(fmt.Sprintf("min_running=%d", c.VMAlerts.MinRunning)), + ) + fmt.Fprintf(a.out, "%s %s / %s\n", + colorLabel("Routing: "), + colorWarn(fmt.Sprintf("critical_immediate=%t", c.AlertRouting.CriticalImmediate)), + colorWarn(fmt.Sprintf("warning_batch=%dm", c.AlertRouting.WarningBatchMins)), + ) + fmt.Fprintf(a.out, "%s %s / %s / %s\n", + colorLabel("RB trigger: "), + colorWarn(fmt.Sprintf("enabled=%t", c.RunbookTrigger.Enabled)), + colorWarn(fmt.Sprintf("runbook=%s", emptyFallback(c.RunbookTrigger.RunbookID, "-"))), + colorWarn(fmt.Sprintf("cooldown=%dm", c.RunbookTrigger.CooldownMins)), + ) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Tags: "), colorInfo(emptyFallback(strings.Join(c.Tags, ","), "-"))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent: "), agentStatusColored) + if c.Agent.Installed { + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent port: "), colorBlue(fmt.Sprintf("%d", c.Agent.Port))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent token:"), colorDim(printableSecret(c.Agent.Token))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent reqsec:"), colorDim(printableSecret(c.Agent.RequestSecret))) + tlsMode := ternary(c.Agent.TLSEnabled, "enabled", "disabled") + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent TLS: "), colorValue(tlsMode)) + if c.Agent.TLSEnabled { + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent TLS fp:"), colorInfo(emptyFallback(c.Agent.TLSFingerprint, "-"))) + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent bin: "), colorInfo(c.Agent.RemoteBinary)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent conf: "), colorInfo(c.Agent.RemoteConfig)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Agent log: "), colorInfo(c.Agent.RemoteLog)) + } + softwareSummary := c.Software.Summary() + softwareColored := colorOK(softwareSummary) + if softwareSummary == "-" || softwareSummary == "" { + softwareColored = colorCrit("-") + } else if softwareSummary == "none" { + softwareColored = colorWarn(softwareSummary) + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Software: "), softwareColored) + if !c.Software.DetectedAt.IsZero() { + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Soft scan: "), colorDim(c.Software.DetectedAt.Format(time.RFC3339))) + } + if len(c.Software.Versions) > 0 { + keys := make([]string, 0, len(c.Software.Versions)) + for k := range c.Software.Versions { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(a.out, "%s %s\n", colorLabel(fmt.Sprintf("Soft %-6s:", k)), colorInfo(c.Software.Versions[k])) + } + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Created: "), colorDim(c.CreatedAt.Format(time.RFC3339))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Updated: "), colorDim(c.UpdatedAt.Format(time.RFC3339))) + + return 0 +} + +func (a *App) runClusterCurrent(svc *cluster.Service, jsonOut bool) int { + c, err := svc.Current() + if err != nil { + fmt.Fprintln(a.err, "no active cluster") + return 1 + } + c = sanitizeCluster(c, false) + + if jsonOut { + _ = writeJSON(a.out, c) + return 0 + } + + fmt.Fprintf(a.out, "%s %s%s%s%s%s\n", + colorAccent(c.Name), + colorMuted("("), + colorValue(c.User), + colorMuted("@"), + colorBlue(fmt.Sprintf("%s:%d", c.Host, c.Port)), + colorMuted(")"), + ) + return 0 +} + +func (a *App) runClusterUse(svc *cluster.Service, args []string) int { + if len(args) != 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster use ") + return 2 + } + + c, err := svc.Use(args[0]) + if err != nil { + fmt.Fprintf(a.err, "set active cluster: %v\n", err) + return 1 + } + + fmt.Fprintf(a.out, "Active cluster is now %q\n", c.Name) + return 0 +} + +func (a *App) runClusterDisconnect(svc *cluster.Service, args []string) int { + if len(args) != 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster disconnect ") + return 2 + } + + c, err := svc.Disconnect(args[0]) + if err != nil { + fmt.Fprintf(a.err, "disconnect cluster: %v\n", err) + return 1 + } + + fmt.Fprintf(a.out, "Disconnected cluster %q\n", c.Name) + return 0 +} + +func (a *App) runClusterSetAuth(svc *cluster.Service, args []string) int { + fs := flag.NewFlagSet("cluster set-auth", flag.ContinueOnError) + fs.SetOutput(a.err) + + auth := fs.String("auth", "", "SSH auth method: key|password (optional, keeps current if empty)") + password := fs.String("password", "", "New SSH password") + storePassword := fs.Bool("store-password", false, "Persist password in encrypted local store") + clearPassword := fs.Bool("clear-password", false, "Remove stored password") + keyPath := fs.String("key-path", "", "New SSH private key path") + clearKeyPath := fs.Bool("clear-key-path", false, "Remove stored key path") + keyPass := fs.String("key-passphrase", "", "New key passphrase") + keyPassFile := fs.String("key-passphrase-file", "", "File containing new key passphrase") + storeKeyPass := fs.Bool("store-key-passphrase", false, "Persist key passphrase in encrypted local store") + storeKeyPassFile := fs.Bool("store-key-passphrase-file", false, "Persist key passphrase file path in encrypted local store") + clearKeyPass := fs.Bool("clear-key-passphrase", false, "Remove stored key passphrase") + insecure := fs.String("insecure-host-key", "", "Override host-key verification: on|off (empty keeps current)") + transport := fs.String("type", "", "Network transport: direct|ipfabric (empty keeps current)") + + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster set-auth [flags]") + return 2 + } + selector = fs.Arg(0) + } + if strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster set-auth [flags]") + return 2 + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster set-auth [flags]") + return 2 + } + + opts := cluster.UpdateAuthOptions{} + if strings.TrimSpace(*auth) != "" { + opts.AuthMethod = cluster.AuthMethod(strings.ToLower(strings.TrimSpace(*auth))) + } + + // Track flag presence to distinguish "unset" from "set to empty". + seen := map[string]bool{} + fs.Visit(func(f *flag.Flag) { seen[f.Name] = true }) + resolvedKeyPass, err := readSecretValueFileFlag("key passphrase", *keyPass, *keyPassFile) + if err != nil { + fmt.Fprintf(a.err, "set auth: %v\n", err) + return 2 + } + + if seen["password"] || *storePassword { + opts.SetPassword = true + opts.Password = *password + opts.StorePassword = *storePassword + } + if *clearPassword { + opts.ClearPassword = true + } + if seen["key-path"] || *clearKeyPath { + opts.SetKeyPath = true + if *clearKeyPath { + opts.KeyPath = "" + } else { + opts.KeyPath = *keyPath + } + } + if seen["key-passphrase"] || seen["key-passphrase-file"] || *storeKeyPass { + opts.SetKeyPassphrase = true + opts.KeyPassphrase = resolvedKeyPass + opts.StoreKeyPassphrase = *storeKeyPass + } + if seen["key-passphrase-file"] || *storeKeyPassFile { + opts.SetKeyPassphraseFile = true + opts.KeyPassphraseFile = *keyPassFile + opts.StoreKeyPassphraseFile = *storeKeyPassFile + } + if *clearKeyPass { + opts.ClearKeyPassphrase = true + } + + switch strings.ToLower(strings.TrimSpace(*insecure)) { + case "": + case "on", "true", "yes", "1": + opts.SetInsecureHostKey = true + opts.InsecureHostKey = true + case "off", "false", "no", "0": + opts.SetInsecureHostKey = true + opts.InsecureHostKey = false + default: + fmt.Fprintf(a.err, "invalid value for --insecure-host-key: %q (use on|off)\n", *insecure) + return 2 + } + + if strings.TrimSpace(*transport) != "" { + mode, tErr := parseTransportFlag(*transport) + if tErr != nil { + fmt.Fprintf(a.err, "%v\n", tErr) + return 2 + } + opts.SetTransport = true + opts.Transport = mode + } + + c, err := svc.UpdateAuth(selector, opts) + if err != nil { + fmt.Fprintf(a.err, "set-auth: %v\n", err) + return 1 + } + + fmt.Fprintf(a.out, "%s %s %s %s %s %s\n", + colorLabel("Updated auth for:"), + colorAccent(c.Name), + colorLabel("method:"), + colorValue(string(c.AuthMethod)), + colorLabel("stored:"), + colorValue(authStoredSummary(c)), + ) + if c.InsecureHostKey { + fmt.Fprintln(a.err, "warning: insecure host-key mode is ON (MITM risk). Use --insecure-host-key off to restore strict verification.") + } + return 0 +} + +func parseTransportFlag(raw string) (cluster.TransportMode, error) { + v := strings.ToLower(strings.TrimSpace(raw)) + switch v { + case "", "direct": + return cluster.TransportDirect, nil + case "ipfabric", "ip-fabric", "ip_fabric": + return cluster.TransportIPFabric, nil + default: + return "", fmt.Errorf("invalid --type %q (allowed: direct, ipfabric)", raw) + } +} + +func readSecretValueFileFlag(label, inlineValue, filePath string) (string, error) { + if strings.TrimSpace(filePath) == "" { + return inlineValue, nil + } + if inlineValue != "" { + return "", fmt.Errorf("--%s and --%s-file cannot be used together", strings.ReplaceAll(label, " ", "-"), strings.ReplaceAll(label, " ", "-")) + } + expanded, err := expandCLIPath(filePath) + if err != nil { + return "", fmt.Errorf("read %s file: %w", label, err) + } + data, err := os.ReadFile(expanded) + if err != nil { + return "", fmt.Errorf("read %s file: %w", label, err) + } + value := strings.TrimRight(string(data), "\r\n") + if value == "" { + return "", fmt.Errorf("%s file is empty", label) + } + return value, nil +} + +func expandCLIPath(path string) (string, error) { + p := strings.TrimSpace(path) + if p == "" { + return "", errors.New("empty path") + } + if strings.HasPrefix(p, "~") { + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + if p == "~" { + p = home + } else { + p = filepath.Join(home, strings.TrimPrefix(p, "~/")) + } + } + return p, nil +} + +func authStoredSummary(c cluster.Cluster) string { + parts := []string{} + if strings.TrimSpace(c.Password) != "" { + parts = append(parts, "password") + } + if strings.TrimSpace(c.KeyPath) != "" { + parts = append(parts, "key-path") + } + if strings.TrimSpace(c.KeyPassphrase) != "" { + parts = append(parts, "key-passphrase") + } + if strings.TrimSpace(c.KeyPassphraseFile) != "" { + parts = append(parts, "key-passphrase-file") + } + if len(parts) == 0 { + return "none" + } + return strings.Join(parts, ",") +} + +func repoTunnelDisplay(proxy string) string { + v := strings.TrimSpace(proxy) + v = strings.TrimPrefix(v, "http://") + v = strings.TrimPrefix(v, "https://") + return emptyFallback(v, "enabled") +} + +func (a *App) runClusterOpenSSH(svc *cluster.Service, args []string) int { + selector := "" + if len(args) > 0 { + first := strings.TrimSpace(args[0]) + if first != "" && !strings.HasPrefix(first, "-") { + selector = first + } + } + + c, err := svc.Get(selector) + if err != nil { + if errors.Is(err, cluster.ErrNoActiveCluster) { + fmt.Fprintln(a.err, "no active cluster; pass name or run `cluster use ` first") + return 1 + } + fmt.Fprintf(a.err, "openssh: %v\n", err) + return 1 + } + + stdinFd := int(os.Stdin.Fd()) + if !term.IsTerminal(stdinFd) { + fmt.Fprintln(a.err, "openssh: stdin is not a terminal; interactive shell requires a tty") + return 1 + } + + fmt.Fprintf(a.out, "%s %s\r\n", + colorLabel("Opening SSH:"), + colorAccent(fmt.Sprintf("%s@%s:%d", c.User, c.Host, c.Port)), + ) + + width, height, err := term.GetSize(stdinFd) + if err != nil || width <= 0 || height <= 0 { + width, height = 120, 32 + } + + oldState, err := term.MakeRaw(stdinFd) + if err != nil { + fmt.Fprintf(a.err, "openssh: raw mode: %v\n", err) + return 1 + } + defer func() { _ = term.Restore(stdinFd, oldState) }() + + resizeCh := make(chan cluster.InteractiveShellSize, 4) + resizeCh <- cluster.InteractiveShellSize{Width: width, Height: height} + + sigCh := installWinchHandler(stdinFd, resizeCh) + defer closeWinchHandler(sigCh, resizeCh) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + opts := cluster.InteractiveShellOptions{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + Term: os.Getenv("TERM"), + Width: width, + Height: height, + Resize: resizeCh, + } + + if err := svc.OpenInteractiveShell(ctx, c.ID, opts); err != nil { + _ = term.Restore(stdinFd, oldState) + fmt.Fprintf(a.err, "\r\nopenssh: %v\r\n", err) + return 1 + } + return 0 +} + +func (a *App) runClusterExec(svc *cluster.Service, args []string, jsonOut bool) int { + selector, rest := splitLeadingSelector(args) + if strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster exec -- ") + return 2 + } + if len(rest) > 0 && rest[0] == "--" { + rest = rest[1:] + } + if len(rest) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster exec -- ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + out, err := svc.RunRemoteShell(ctx, selector, strings.Join(rest, " ")) + if err != nil { + fmt.Fprintf(a.err, "exec: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 +} + +func (a *App) runClusterPing(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster ping", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Password override for SSH auth") + keyPassphrase := fs.String("key-passphrase", "", "Key passphrase override for SSH auth") + keyPassphraseFile := fs.String("key-passphrase-file", "", "File containing key passphrase override for SSH auth") + agent := fs.Bool("agent", false, "Ping pxmon-agent instead of SSH") + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster ping [name-or-id] [--agent]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster ping [name-or-id] [--agent]") + return 2 + } + resolvedKeyPassphrase, err := readSecretValueFileFlag("key passphrase", *keyPassphrase, *keyPassphraseFile) + if err != nil { + fmt.Fprintf(a.err, "ssh ping: %v\n", err) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + + if *agent { + result, err := svc.PingAgent(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "agent ping: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, result) + return 0 + } + if result.Reachable && result.StatusCode < 400 { + fmt.Fprintf(a.out, "%s %s %s\n", + colorOK("AGENT OK"), + colorBlue(result.Endpoint), + colorMuted(fmt.Sprintf("[%d]", result.StatusCode)), + ) + return 0 + } + fmt.Fprintf(a.err, "%s %s\n", + colorCrit("AGENT FAILED"), + colorWarn(emptyFallback(result.Error, "unknown error")), + ) + return 1 + } + + result, err := svc.ProbeSSH(ctx, selector, *password, resolvedKeyPassphrase) + if err != nil { + fmt.Fprintf(a.err, "ssh ping: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, result) + return 0 + } + + if result.Reachable { + fmt.Fprintf(a.out, "%s %s %s\n", + colorOK("SSH OK"), + colorBlue(result.Address), + colorMuted(fmt.Sprintf("(%dms)", result.LatencyMS)), + ) + return 0 + } + + fmt.Fprintf(a.err, "%s %s\n", colorCrit("SSH FAILED"), colorWarn(result.Error)) + return 1 +} + +func (a *App) runClusterBootstrap(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster bootstrap", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Password override for SSH auth") + keyPassphrase := fs.String("key-passphrase", "", "Key passphrase override for SSH auth") + keyPassphraseFile := fs.String("key-passphrase-file", "", "File containing key passphrase override for SSH auth") + listen := fs.String("listen", "0.0.0.0:19090", "Agent listen address on remote node") + port := fs.Int("port", 0, "Agent port override") + agentBin := fs.String("agent-bin", "", "Local path to prebuilt pxmon-agent binary") + rotateToken := fs.Bool("rotate-token", false, "Rotate API token while bootstrapping") + allowProbeFail := fs.Bool("allow-agent-probe-fail", false, "Do not fail command if post-install agent probe fails") + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster bootstrap [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster bootstrap [name-or-id]") + return 2 + } + resolvedKeyPassphrase, err := readSecretValueFileFlag("key passphrase", *keyPassphrase, *keyPassphraseFile) + if err != nil { + fmt.Fprintf(a.err, "bootstrap cluster: %v\n", err) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + c, result, err := svc.BootstrapAgent(ctx, cluster.BootstrapOptions{ + Selector: selector, + Password: *password, + KeyPassphrase: resolvedKeyPassphrase, + ListenAddress: *listen, + AgentPort: *port, + LocalAgentBin: *agentBin, + RotateToken: *rotateToken, + AllowAgentProbe: *allowProbeFail, + }) + if err != nil { + fmt.Fprintf(a.err, "bootstrap cluster: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": sanitizeCluster(c, false), + "result": result, + }) + return 0 + } + + fmt.Fprintf(a.out, "Bootstrap complete for %q\n", c.Name) + fmt.Fprintf(a.out, "Remote runtime: %s/%s\n", result.RemoteOS, result.RemoteArch) + fmt.Fprintf(a.out, "Remote PID: %s\n", result.PID) + if result.AgentPing.Reachable && result.AgentPing.StatusCode < 400 { + fmt.Fprintf(a.out, "Agent ping: OK %s [%d]\n", result.AgentPing.Endpoint, result.AgentPing.StatusCode) + } else { + fmt.Fprintf(a.out, "Agent ping: FAILED %s\n", emptyFallback(result.AgentPing.Error, "unknown error")) + } + _ = svc.AppendChange("agent.bootstrap", c.Name, fmt.Sprintf("version=%s", c.Agent.Version)) + return 0 +} + +func (a *App) runClusterAgent(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster agent [name-or-id]") + return 2 + } + + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "status": + fs := flag.NewFlagSet("cluster agent status", flag.ContinueOnError) + fs.SetOutput(a.err) + timeout := fs.Duration("timeout", 1500*time.Millisecond, "Per-node ping timeout") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster agent status [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster agent status [name-or-id]") + return 2 + } + + expectedVersion := svc.ExpectedAgentVersion() + if strings.TrimSpace(selector) != "" { + c, err := svc.Get(selector) + if err != nil { + fmt.Fprintf(a.err, "agent status: %v\n", err) + return 1 + } + status, code := buildAgentStatusRow(svc, c, expectedVersion, *timeout) + if jsonOut { + _ = writeJSON(a.out, status) + return 0 + } + if code != 0 { + fmt.Fprintf(a.err, "%s\n", statusErrorText(status)) + return code + } + printAgentStatusTable(a.out, []map[string]any{status}) + return 0 + } + + clusters, _, err := svc.List() + if err != nil { + fmt.Fprintf(a.err, "agent status: %v\n", err) + return 1 + } + items := make([]map[string]any, 0, len(clusters)) + failed := false + for _, c := range clusters { + row, code := buildAgentStatusRow(svc, c, expectedVersion, *timeout) + if code != 0 { + failed = true + } + items = append(items, row) + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "expected_version": expectedVersion, + "items": items, + }) + if failed { + return 1 + } + return 0 + } + printAgentStatusTable(a.out, items) + if failed { + return 1 + } + return 0 + case "adopt-auth", "sync-auth", "repair-auth": + fs := flag.NewFlagSet("cluster agent adopt-auth", flag.ContinueOnError) + fs.SetOutput(a.err) + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster agent adopt-auth [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster agent adopt-auth [name-or-id]") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + c, ping, err := svc.AdoptAgentAuth(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "agent adopt-auth: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": sanitizeCluster(c, false), + "ping": ping, + }) + return 0 + } + fmt.Fprintf(a.out, "Adopted agent auth for %q\n", c.Name) + if ping.Reachable && ping.StatusCode < 400 { + fmt.Fprintf(a.out, "Agent ping: OK %s [%d]\n", ping.Endpoint, ping.StatusCode) + } else { + fmt.Fprintf(a.out, "Agent ping: FAILED %s\n", emptyFallback(ping.Error, "unknown error")) + } + return 0 + case "versions", "version-list": + items := svc.AgentVersions() + if jsonOut { + _ = writeJSON(a.out, map[string]any{"items": items}) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no known versions)") + return 0 + } + for _, it := range items { + fmt.Fprintf(a.out, "%s", it.Version) + if strings.TrimSpace(it.ReleasedAt) != "" { + fmt.Fprintf(a.out, " (%s)", it.ReleasedAt) + } + fmt.Fprintln(a.out) + for _, feat := range it.Features { + fmt.Fprintf(a.out, " - %s\n", feat) + } + } + return 0 + case "update", "upgrade", "reinstall": + fs := flag.NewFlagSet("cluster agent update", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Password override for SSH auth") + keyPassphrase := fs.String("key-passphrase", "", "Key passphrase override for SSH auth") + keyPassphraseFile := fs.String("key-passphrase-file", "", "File containing key passphrase override for SSH auth") + listen := fs.String("listen", "0.0.0.0:19090", "Agent listen address on remote node") + port := fs.Int("port", 0, "Agent port override") + agentBin := fs.String("agent-bin", "", "Local path to prebuilt pxmon-agent binary") + rotateToken := fs.Bool("rotate-token", false, "Rotate API token during update") + allowProbeFail := fs.Bool("allow-agent-probe-fail", false, "Do not fail command if post-update probe fails") + restartBot := fs.Bool("restart-bot", true, "Restart local telegram bot daemon after update (if enabled)") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster agent update [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster agent update [name-or-id]") + return 2 + } + resolvedKeyPassphrase, err := readSecretValueFileFlag("key passphrase", *keyPassphrase, *keyPassphraseFile) + if err != nil { + fmt.Fprintf(a.err, "agent update: %v\n", err) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + c, result, err := svc.BootstrapAgent(ctx, cluster.BootstrapOptions{ + Selector: selector, + Password: *password, + KeyPassphrase: resolvedKeyPassphrase, + ListenAddress: *listen, + AgentPort: *port, + LocalAgentBin: *agentBin, + RotateToken: *rotateToken, + AllowAgentProbe: *allowProbeFail, + }) + if err != nil { + fmt.Fprintf(a.err, "agent update: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": sanitizeCluster(c, false), + "result": result, + }) + return 0 + } + fmt.Fprintf(a.out, "Agent updated for %q\n", c.Name) + fmt.Fprintf(a.out, "Remote runtime: %s/%s\n", result.RemoteOS, result.RemoteArch) + fmt.Fprintf(a.out, "PID: %s\n", result.PID) + fmt.Fprintf(a.out, "Version: node=%s local=%s\n", emptyFallback(c.Agent.Version, "unknown"), svc.ExpectedAgentVersion()) + if result.AgentPing.Reachable && result.AgentPing.StatusCode < 400 { + fmt.Fprintf(a.out, "Agent ping: OK %s [%d]\n", result.AgentPing.Endpoint, result.AgentPing.StatusCode) + } else { + fmt.Fprintf(a.out, "Agent ping: FAILED %s\n", emptyFallback(result.AgentPing.Error, "unknown error")) + } + if *restartBot { + if tgCfg, tgErr := svc.GetTelegram(); tgErr == nil && tgCfg.Enabled { + if pid, _, rbErr := restartTelegramBotDaemon(svc, svc.ConfigPath(), telegramBotDefaultPoll); rbErr != nil { + fmt.Fprintf(a.err, "agent update: warning: failed to restart telegram bot daemon: %v\n", rbErr) + } else if !jsonOut { + fmt.Fprintf(a.out, "Telegram bot daemon restarted (pid %d)\n", pid) + } + } + } + _ = svc.AppendChange("agent.update", c.Name, fmt.Sprintf("version=%s", c.Agent.Version)) + return 0 + default: + fmt.Fprintf(a.err, "unknown agent subcommand %q\n", args[0]) + fmt.Fprintln(a.err, "usage: pxmon cluster agent [name-or-id]") + return 2 + } +} + +func (a *App) runClusterRepoTunnel(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + a.printRepoTunnelHelp() + return 0 + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "enable", "on": + fs := flag.NewFlagSet("cluster repo-tunnel enable", flag.ContinueOnError) + fs.SetOutput(a.err) + gateway := fs.String("gateway", "", "Repo gateway proxy endpoint host[:port] (default port 3128)") + gatewayIP := fs.String("gateway-ip", "", "Resolved gateway IP for ip rule; skips DNS on node") + table := fs.Int("table", 0, "Routing table used for gateway egress (required)") + priority := fs.Int("priority", 0, "Optional ip rule priority") + manager := fs.String("manager", "auto", "Package manager: auto|apt|dnf|yum") + noRule := fs.Bool("no-rule", false, "Only configure package proxy; do not add ip rule") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel enable --gateway --table ") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 || strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel enable --gateway --table
") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + out, err := svc.RepoTunnelEnable(ctx, selector, cluster.RepoTunnelOptions{ + Gateway: *gateway, + GatewayIP: *gatewayIP, + Table: *table, + Priority: *priority, + PackageManager: *manager, + NoRule: *noRule, + }) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel enable: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 + case "disable", "off": + fs := flag.NewFlagSet("cluster repo-tunnel disable", flag.ContinueOnError) + fs.SetOutput(a.err) + gateway := fs.String("gateway", "", "Repo gateway proxy endpoint host[:port]") + gatewayIP := fs.String("gateway-ip", "", "Resolved gateway IP for ip rule; skips DNS on node") + table := fs.Int("table", 0, "Routing table used for gateway egress") + noRule := fs.Bool("no-rule", false, "Only remove package proxy; do not delete ip rule") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel disable --gateway --table
") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 || strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel disable --gateway --table
") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + out, err := svc.RepoTunnelDisable(ctx, selector, cluster.RepoTunnelOptions{ + Gateway: *gateway, + GatewayIP: *gatewayIP, + Table: *table, + NoRule: *noRule, + }) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel disable: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 + case "status": + fs := flag.NewFlagSet("cluster repo-tunnel status", flag.ContinueOnError) + fs.SetOutput(a.err) + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel status ") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 || strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel status ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + out, err := svc.RepoTunnelStatus(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel status: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 + case "install", "run": + fs := flag.NewFlagSet("cluster repo-tunnel install", flag.ContinueOnError) + fs.SetOutput(a.err) + gateway := fs.String("gateway", "", "Repo gateway proxy endpoint host[:port] (default port 3128)") + gatewayIP := fs.String("gateway-ip", "", "Resolved gateway IP for ip rule; skips DNS on node") + table := fs.Int("table", 0, "Routing table used for gateway egress (required)") + priority := fs.Int("priority", 0, "Optional ip rule priority") + manager := fs.String("manager", "auto", "Package manager: auto|apt|dnf|yum") + noRule := fs.Bool("no-rule", false, "Only configure package proxy; do not add ip rule") + keep := fs.Bool("keep-enabled", false, "Keep repo tunnel enabled after command finishes") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if strings.TrimSpace(selector) == "" && fs.NArg() > 0 { + selector = fs.Arg(0) + } + if strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel install --gateway --table
-- ") + return 2 + } + cmdArgs := fs.Args() + if len(cmdArgs) > 0 && cmdArgs[0] == selector { + cmdArgs = cmdArgs[1:] + } + if len(cmdArgs) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel install --gateway --table
-- ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + out, err := svc.RepoTunnelInstall(ctx, selector, cluster.RepoTunnelOptions{ + Gateway: *gateway, + GatewayIP: *gatewayIP, + Table: *table, + Priority: *priority, + PackageManager: *manager, + Command: strings.Join(cmdArgs, " "), + KeepEnabled: *keep, + NoRule: *noRule, + }) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel install: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 + case "gateway-script": + fs := flag.NewFlagSet("cluster repo-tunnel gateway-script", flag.ContinueOnError) + fs.SetOutput(a.err) + port := fs.Int("port", 3128, "Squid listen port") + allows := multiFlag{} + fs.Var(&allows, "allow", "Allowed node source CIDR/IP (repeatable)") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + script, err := cluster.RepoTunnelGatewayScript(*port, allows) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel gateway-script: %v\n", err) + return 1 + } + fmt.Fprintln(a.out, script) + return 0 + case "gateway-setup": + fs := flag.NewFlagSet("cluster repo-tunnel gateway-setup", flag.ContinueOnError) + fs.SetOutput(a.err) + port := fs.Int("port", 3128, "Squid listen port") + allows := multiFlag{} + fs.Var(&allows, "allow", "Allowed node source CIDR/IP (repeatable)") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel gateway-setup --allow ") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 || strings.TrimSpace(selector) == "" { + fmt.Fprintln(a.err, "usage: pxmon cluster repo-tunnel gateway-setup --allow ") + return 2 + } + script, err := cluster.RepoTunnelGatewayScript(*port, allows) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel gateway-setup: %v\n", err) + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + out, err := svc.RunRemoteShell(ctx, selector, script) + if err != nil { + fmt.Fprintf(a.err, "repo-tunnel gateway-setup: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"output": out}) + } else { + fmt.Fprint(a.out, out) + } + return 0 + case "help", "--help", "-h": + a.printRepoTunnelHelp() + return 0 + default: + fmt.Fprintf(a.err, "unknown repo-tunnel command %q\n\n", args[0]) + a.printRepoTunnelHelp() + return 2 + } +} + +func (a *App) printRepoTunnelHelp() { + fmt.Fprintln(a.out, "pxmon cluster repo-tunnel commands:") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, " gateway-script Print a squid setup script for a gateway VM") + fmt.Fprintln(a.out, " gateway-setup Install/configure squid on a managed gateway VM") + fmt.Fprintln(a.out, " enable Add ip rule and package proxy config on an ipfabric node") + fmt.Fprintln(a.out, " install Enable tunnel, run package command, then disable it") + fmt.Fprintln(a.out, " disable Remove package proxy config and matching ip rule") + fmt.Fprintln(a.out, " status Show repo tunnel config and ip rules") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Examples:") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel gateway-script --allow 198.51.100.20/32 --port 3128") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel gateway-setup repo-vm --allow 198.51.100.20/32 --port 3128") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel enable edge-node-1 --gateway 203.0.113.10:3128 --table 1010 --manager dnf") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel install edge-node-1 --gateway 203.0.113.10:3128 --table 1010 -- dnf install -y curl jq") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel disable edge-node-1 --gateway 203.0.113.10:3128 --table 1010") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Notes:") + fmt.Fprintln(a.out, " --table is intentionally required because ipfabric routing tables differ between nodes.") + fmt.Fprintln(a.out, " Use --gateway-ip if the node cannot resolve the gateway hostname before proxy is enabled.") +} + +func buildAgentStatusRow(svc *cluster.Service, c cluster.Cluster, expectedVersion string, timeout time.Duration) (map[string]any, int) { + row := map[string]any{ + "cluster": c.Name, + "id": c.ID, + "installed": c.Agent.Installed, + "expected_version": expectedVersion, + "node_version": strings.TrimSpace(c.Agent.Version), + "status": "NOT_INSTALLED", + "online": false, + "error": "", + } + if !c.Agent.Installed { + return row, 1 + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + ping, err := svc.PingAgent(ctx, c.ID) + if err != nil { + row["status"] = "OFFLINE" + row["error"] = err.Error() + return row, 1 + } + row["online"] = ping.Reachable && ping.StatusCode < 400 + if strings.TrimSpace(ping.Version) != "" { + row["node_version"] = strings.TrimSpace(ping.Version) + } + row["endpoint"] = ping.Endpoint + row["http_status"] = ping.StatusCode + + nodeVersion := strings.TrimSpace(fmt.Sprintf("%v", row["node_version"])) + if !ping.Reachable || ping.StatusCode >= 400 { + row["status"] = "OFFLINE" + if strings.TrimSpace(ping.Error) != "" { + row["error"] = ping.Error + } + return row, 1 + } + if nodeVersion == "" { + row["status"] = "UNKNOWN_VERSION" + return row, 1 + } + if strings.TrimSpace(expectedVersion) == "" || nodeVersion == expectedVersion { + row["status"] = "OK" + return row, 0 + } + row["status"] = "MISMATCH" + row["error"] = fmt.Sprintf("node=%s local=%s", nodeVersion, expectedVersion) + return row, 1 +} + +func statusErrorText(row map[string]any) string { + clusterName := strings.TrimSpace(fmt.Sprintf("%v", row["cluster"])) + status := strings.TrimSpace(fmt.Sprintf("%v", row["status"])) + errText := strings.TrimSpace(fmt.Sprintf("%v", row["error"])) + if errText == "" { + errText = status + } + return fmt.Sprintf("%s: %s", clusterName, errText) +} + +func printAgentStatusTable(w io.Writer, items []map[string]any) { + tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", + colorHeader("CLUSTER"), + colorHeader("STATUS"), + colorHeader("ONLINE"), + colorHeader("NODE_VERSION"), + colorHeader("LOCAL_VERSION"), + colorHeader("DETAIL"), + ) + for _, row := range items { + clusterName := strings.TrimSpace(fmt.Sprintf("%v", row["cluster"])) + status := strings.TrimSpace(fmt.Sprintf("%v", row["status"])) + online := strings.TrimSpace(fmt.Sprintf("%v", row["online"])) + nodeVersion := strings.TrimSpace(fmt.Sprintf("%v", row["node_version"])) + localVersion := strings.TrimSpace(fmt.Sprintf("%v", row["expected_version"])) + detail := strings.TrimSpace(fmt.Sprintf("%v", row["error"])) + if detail == "" { + detail = "-" + } + if nodeVersion == "" { + nodeVersion = "-" + } + if localVersion == "" { + localVersion = "-" + } + + renderStatus := status + switch status { + case "OK": + renderStatus = colorOK(status) + case "MISMATCH", "UNKNOWN_VERSION": + renderStatus = colorWarn(status) + default: + renderStatus = colorCrit(status) + } + onlineColored := colorCrit(online) + if online == "true" { + onlineColored = colorOK(online) + } + detailColored := colorDim(detail) + if detail != "-" && status != "OK" { + detailColored = colorWarn(detail) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", + colorAccent(clusterName), + renderStatus, + onlineColored, + colorInfo(nodeVersion), + colorBlue(localVersion), + detailColored, + ) + } + _ = tw.Flush() +} + +func (a *App) runClusterStats(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster stats", flag.ContinueOnError) + fs.SetOutput(a.err) + once := fs.Bool("once", false, "Print one snapshot and exit") + interval := fs.Duration("interval", 2*time.Second, "Refresh interval for interactive mode") + iface := fs.String("iface", "", "Initial network interface for monitor view") + cpuWarn := fs.Float64("cpu-warn", 0, "Override CPU warning threshold percent") + ramWarn := fs.Float64("ram-warn", 0, "Override RAM warning threshold percent") + swapWarn := fs.Float64("swap-warn", 0, "Override swap warning threshold percent") + diskWarn := fs.Float64("disk-warn", 0, "Override disk warning threshold percent") + netWarn := fs.Float64("net-warn-mbps", 0, "Override network warning threshold Mbps") + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster stats [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster stats [name-or-id]") + return 2 + } + + policy, err := svc.GetAlertPolicy(selector) + if err != nil { + if errors.Is(err, cluster.ErrNoActiveCluster) { + fmt.Fprintln(a.err, "no active cluster") + return 1 + } + fmt.Fprintf(a.err, "load alert policy: %v\n", err) + return 1 + } + if *cpuWarn > 0 { + policy.CPUWarnPercent = *cpuWarn + } + if *ramWarn > 0 { + policy.RAMWarnPercent = *ramWarn + } + if *swapWarn > 0 { + policy.SwapWarnPercent = *swapWarn + } + if *diskWarn > 0 { + policy.DiskWarnPercent = *diskWarn + } + if *netWarn > 0 { + policy.NetWarnMbps = *netWarn + } + + if jsonOut || *once { + return a.runClusterStatsOnce(svc, selector, jsonOut) + } + + return a.runClusterMonitor(svc, selector, MonitorOptions{ + Interval: *interval, + InitialIface: strings.TrimSpace(*iface), + AlertPolicy: policy, + InitialView: "overview", + }) +} + +func (a *App) runClusterStatsOnce(svc *cluster.Service, selector string, jsonOut bool) int { + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + + stats, err := svc.AgentStatsTyped(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "cluster stats: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, stats) + return 0 + } + + printStatsSnapshot(a.out, stats) + return 0 +} + +func (a *App) runClusterSoftware(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster software [name-or-id]") + return 2 + } + + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + c, err := svc.Get(selector) + if err != nil { + fmt.Fprintf(a.err, "software show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, c.Software) + return 0 + } + detected := "never" + detectedColor := colorWarn + if !c.Software.DetectedAt.IsZero() { + detected = c.Software.DetectedAt.Format(time.RFC3339) + detectedColor = colorDim + } + summary := c.Software.Summary() + summaryColored := colorOK(summary) + if summary == "-" || summary == "" { + summaryColored = colorCrit("-") + } else if summary == "none" { + summaryColored = colorWarn(summary) + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Cluster: "), colorAccent(c.Name)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Detected:"), detectedColor(detected)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Support: "), summaryColored) + if len(c.Software.Versions) > 0 { + keys := make([]string, 0, len(c.Software.Versions)) + for k := range c.Software.Versions { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(a.out, "%s %s\n", colorLabel(fmt.Sprintf("Version %-4s:", k)), colorInfo(c.Software.Versions[k])) + } + } + return 0 + case "scan", "refresh": + fs := flag.NewFlagSet("cluster software scan", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Password override for SSH auth") + keyPassphrase := fs.String("key-passphrase", "", "Key passphrase override for SSH auth") + keyPassphraseFile := fs.String("key-passphrase-file", "", "File containing key passphrase override for SSH auth") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster software scan [name-or-id]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster software scan [name-or-id]") + return 2 + } + resolvedKeyPassphrase, err := readSecretValueFileFlag("key passphrase", *keyPassphrase, *keyPassphraseFile) + if err != nil { + fmt.Fprintf(a.err, "software scan: %v\n", err) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + c, info, err := svc.SoftwareScan(ctx, selector, *password, resolvedKeyPassphrase) + if err != nil { + fmt.Fprintf(a.err, "software scan: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": c.Name, + "software": info, + }) + return 0 + } + fmt.Fprintf(a.out, "Software scan complete for %q\n", c.Name) + fmt.Fprintf(a.out, "Support: %s\n", info.Summary()) + return 0 + default: + fmt.Fprintf(a.err, "unknown software subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterAlert(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster alert [name-or-id]") + return 2 + } + + switch args[0] { + case "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + policy, err := svc.GetAlertPolicy(selector) + if err != nil { + fmt.Fprintf(a.err, "alert show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, policy) + return 0 + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("CPU warn: "), colorWarn(fmt.Sprintf("%.1f%%", policy.CPUWarnPercent))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("RAM warn: "), colorWarn(fmt.Sprintf("%.1f%%", policy.RAMWarnPercent))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Swap warn:"), colorWarn(fmt.Sprintf("%.1f%%", policy.SwapWarnPercent))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Disk warn:"), colorWarn(fmt.Sprintf("%.1f%%", policy.DiskWarnPercent))) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Net warn: "), colorWarn(fmt.Sprintf("%.1f Mbps", policy.NetWarnMbps))) + if policy.NetSustainEnabled { + iface := strings.TrimSpace(policy.NetSustainIface) + if iface == "" { + iface = "(filtered)" + } + include := "-" + exclude := "-" + if len(policy.NetSustainInclude) > 0 { + include = strings.Join(policy.NetSustainInclude, ",") + } + if len(policy.NetSustainExclude) > 0 { + exclude = strings.Join(policy.NetSustainExclude, ",") + } + fmt.Fprintf(a.out, "%s enabled (iface=%s threshold=%.1f Mbps window=%d min cooldown=%d min)\n", + colorLabel("Net sustain:"), + iface, + policy.NetSustainMbps, + policy.NetSustainMinutes, + policy.NetSustainCooldownMins, + ) + fmt.Fprintf(a.out, "%s include=%s exclude=%s\n", colorLabel("Net filter: "), include, exclude) + } else { + fmt.Fprintf(a.out, "%s disabled\n", colorLabel("Net sustain:")) + } + return 0 + case "set": + fs := flag.NewFlagSet("cluster alert set", flag.ContinueOnError) + fs.SetOutput(a.err) + cpuWarn := fs.Float64("cpu", 0, "CPU warning threshold percent") + ramWarn := fs.Float64("ram", 0, "RAM warning threshold percent") + swapWarn := fs.Float64("swap", 0, "Swap warning threshold percent") + diskWarn := fs.Float64("disk", 0, "Disk warning threshold percent") + netWarn := fs.Float64("net-mbps", 0, "Network warning threshold Mbps") + netSustainEnabled := fs.Bool("net-sustain-enabled", false, "Enable sustained network threshold alert") + netSustainIface := fs.String("net-sustain-iface", "", "Interface name for sustained network alert (default: auto)") + netSustainInclude := fs.String("net-sustain-include", "", "CSV substring filter: monitor only matching interfaces (e.g. net0,vm)") + netSustainExclude := fs.String("net-sustain-exclude", "", "CSV substring filter: skip matching interfaces") + netSustainMbps := fs.Float64("net-sustain-mbps", 0, "Sustained network threshold Mbps") + netSustainMins := fs.Int("net-sustain-mins", 0, "Sustained network evaluation window in minutes") + netSustainCooldown := fs.Int("net-sustain-cooldown-mins", 0, "Cooldown between sustained network notifications") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster alert set [name-or-id] [--cpu ...]") + return 2 + } + selector = fs.Arg(0) + } + if fs.NArg() > 1 { + fmt.Fprintln(a.err, "usage: pxmon cluster alert set [name-or-id] [--cpu ...]") + return 2 + } + + policy, err := svc.GetAlertPolicy(selector) + if err != nil { + fmt.Fprintf(a.err, "alert set: %v\n", err) + return 1 + } + if *cpuWarn > 0 { + policy.CPUWarnPercent = *cpuWarn + } + if *ramWarn > 0 { + policy.RAMWarnPercent = *ramWarn + } + if *swapWarn > 0 { + policy.SwapWarnPercent = *swapWarn + } + if *diskWarn > 0 { + policy.DiskWarnPercent = *diskWarn + } + if *netWarn > 0 { + policy.NetWarnMbps = *netWarn + } + seenSustainEnabled := false + seenSustainFields := false + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "net-sustain-enabled": + seenSustainEnabled = true + policy.NetSustainEnabled = *netSustainEnabled + case "net-sustain-iface": + seenSustainFields = true + policy.NetSustainIface = strings.TrimSpace(*netSustainIface) + case "net-sustain-include": + seenSustainFields = true + policy.NetSustainInclude = parseTagsCSV(*netSustainInclude) + case "net-sustain-exclude": + seenSustainFields = true + policy.NetSustainExclude = parseTagsCSV(*netSustainExclude) + case "net-sustain-mbps": + seenSustainFields = true + if *netSustainMbps > 0 { + policy.NetSustainMbps = *netSustainMbps + } + case "net-sustain-mins": + seenSustainFields = true + if *netSustainMins > 0 { + policy.NetSustainMinutes = *netSustainMins + } + case "net-sustain-cooldown-mins": + seenSustainFields = true + if *netSustainCooldown > 0 { + policy.NetSustainCooldownMins = *netSustainCooldown + } + } + }) + if seenSustainFields && !seenSustainEnabled { + policy.NetSustainEnabled = true + } + + c, err := svc.SetAlertPolicy(selector, policy) + if err != nil { + fmt.Fprintf(a.err, "alert set: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": c.Name, + "policy": c.Alerts, + }) + return 0 + } + fmt.Fprintf(a.out, "Updated alerts for %q\n", c.Name) + fmt.Fprintf(a.out, "CPU %.1f%%, RAM %.1f%%, Swap %.1f%%, Disk %.1f%%, Net %.1f Mbps\n", + c.Alerts.CPUWarnPercent, + c.Alerts.RAMWarnPercent, + c.Alerts.SwapWarnPercent, + c.Alerts.DiskWarnPercent, + c.Alerts.NetWarnMbps, + ) + if c.Alerts.NetSustainEnabled { + iface := strings.TrimSpace(c.Alerts.NetSustainIface) + if iface == "" { + iface = "(filtered)" + } + fmt.Fprintf(a.out, "Net sustain: enabled (iface=%s threshold=%.1f Mbps window=%d min cooldown=%d min)\n", + iface, + c.Alerts.NetSustainMbps, + c.Alerts.NetSustainMinutes, + c.Alerts.NetSustainCooldownMins, + ) + include := "-" + exclude := "-" + if len(c.Alerts.NetSustainInclude) > 0 { + include = strings.Join(c.Alerts.NetSustainInclude, ",") + } + if len(c.Alerts.NetSustainExclude) > 0 { + exclude = strings.Join(c.Alerts.NetSustainExclude, ",") + } + fmt.Fprintf(a.out, "Net filter: include=%s exclude=%s\n", include, exclude) + } else { + fmt.Fprintln(a.out, "Net sustain: disabled") + } + return 0 + default: + fmt.Fprintf(a.err, "unknown alert subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runLocker(args []string, configPath string, jsonOut bool) int { + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + + if len(args) == 0 { + args = []string{"status"} + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "status": + cfg, err := svc.GetLocker() + if err != nil { + fmt.Fprintf(a.err, "locker status: %v\n", err) + return 1 + } + locked, expiresAt, err := svc.IsLocked() + if err != nil { + fmt.Fprintf(a.err, "locker status: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "enabled": cfg.Enabled, + "password_set": strings.TrimSpace(cfg.PasswordHash) != "", + "locked": locked, + "session_until": expiresAt, + }) + return 0 + } + boolColor := func(v bool) string { + if v { + return colorOK(fmt.Sprintf("%t", v)) + } + return colorCrit(fmt.Sprintf("%t", v)) + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Enabled: "), boolColor(cfg.Enabled)) + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Password set:"), boolColor(strings.TrimSpace(cfg.PasswordHash) != "")) + lockedColored := colorOK(fmt.Sprintf("%t", locked)) + if locked { + lockedColored = colorWarn(fmt.Sprintf("%t", locked)) + } + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Locked: "), lockedColored) + if !expiresAt.IsZero() { + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Session until:"), colorInfo(expiresAt.Format(time.RFC3339))) + } + return 0 + case "set": + fs := flag.NewFlagSet("locker set", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Locker password") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + pass := strings.TrimSpace(*password) + if pass == "" { + var readErr error + pass, readErr = readPasswordPrompt("Enter locker password: ") + if readErr != nil { + fmt.Fprintf(a.err, "locker set: %v\n", readErr) + return 1 + } + } + cfg, err := svc.SetLockerPassword(pass) + if err != nil { + fmt.Fprintf(a.err, "locker set: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "enabled": cfg.Enabled, + "password_set": strings.TrimSpace(cfg.PasswordHash) != "", + }) + return 0 + } + fmt.Fprintln(a.out, "Locker password set. Locker is enabled.") + return 0 + case "unlock": + fs := flag.NewFlagSet("locker unlock", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Locker password") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + pass := strings.TrimSpace(*password) + if pass == "" { + var readErr error + pass, readErr = readPasswordPrompt("Enter locker password: ") + if readErr != nil { + fmt.Fprintf(a.err, "locker unlock: %v\n", readErr) + return 1 + } + } + if err := svc.UnlockLocker(pass); err != nil { + fmt.Fprintf(a.err, "locker unlock: %v\n", err) + return 1 + } + if !jsonOut { + fmt.Fprintln(a.out, "Unlocked for 6 hours.") + } + return 0 + case "lock": + if err := svc.LockNow(); err != nil { + fmt.Fprintf(a.err, "locker lock: %v\n", err) + return 1 + } + if !jsonOut { + fmt.Fprintln(a.out, "Locked.") + } + return 0 + case "disable": + if _, err := svc.SetLockerEnabled(false); err != nil { + fmt.Fprintf(a.err, "locker disable: %v\n", err) + return 1 + } + if !jsonOut { + fmt.Fprintln(a.out, "Locker disabled.") + } + return 0 + case "logs": + fs := flag.NewFlagSet("locker logs", flag.ContinueOnError) + fs.SetOutput(a.err) + tail := fs.Int("tail", 200, "Number of last log lines") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon locker logs [--tail 200]") + return 2 + } + lines, err := readLastLines(store.LockerAuditPath(), *tail) + if err != nil { + fmt.Fprintf(a.err, "locker logs: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "path": store.LockerAuditPath(), + "lines": lines, + }) + return 0 + } + fmt.Fprintf(a.out, "Log file: %s\n", store.LockerAuditPath()) + if len(lines) == 0 { + fmt.Fprintln(a.out, "(log is empty)") + return 0 + } + for _, line := range lines { + fmt.Fprintln(a.out, line) + } + return 0 + default: + fmt.Fprintln(a.err, "usage: pxmon locker ") + return 2 + } +} + +func readPasswordPrompt(prompt string) (string, error) { + fmt.Fprint(os.Stdout, prompt) + fd := int(os.Stdin.Fd()) + if !term.IsTerminal(fd) { + return "", errors.New("password not provided and stdin is not a terminal") + } + raw, err := term.ReadPassword(fd) + fmt.Fprintln(os.Stdout) + if err != nil { + return "", err + } + return strings.TrimSpace(string(raw)), nil +} + +func (a *App) printRootHelp() { + fmt.Fprintln(a.out, "PXmon (Phylex Monitor) - SSH cluster manager + node agent bootstrap") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Usage:") + fmt.Fprintln(a.out, " pxmon [--config path] [--json] ") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Commands:") + fmt.Fprintln(a.out, " cluster Manage SSH clusters/nodes") + fmt.Fprintln(a.out, " tui Bubble Tea realtime dashboard") + fmt.Fprintln(a.out, " clusters Bubble Tea cluster-overview dashboard") + fmt.Fprintln(a.out, " network Bubble Tea network-focused dashboard") + fmt.Fprintln(a.out, " bot Bot integrations (Telegram)") + fmt.Fprintln(a.out, " locker Global CLI/TUI lock controls") + fmt.Fprintln(a.out, " config Export/import the full registry (clusters + settings)") + fmt.Fprintln(a.out, " explain Fast command discovery with text filter") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Tip: run `pxmon` without arguments in a real terminal to open TUI directly.") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Run 'pxmon cluster help' for cluster subcommands.") +} + +func (a *App) printClusterHelp() { + fmt.Fprintln(a.out, "pxmon cluster commands:") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, " connect Register SSH node") + fmt.Fprintln(a.out, " list List nodes") + fmt.Fprintln(a.out, " current Show active node") + fmt.Fprintln(a.out, " show Show node details") + fmt.Fprintln(a.out, " use Set active node") + fmt.Fprintln(a.out, " ping Check SSH or agent connectivity") + fmt.Fprintln(a.out, " bootstrap Build and install pxmon-agent over SSH") + fmt.Fprintln(a.out, " agent Agent operations (status/update/upgrade)") + fmt.Fprintln(a.out, " stats Interactive realtime monitor (or --once)") + fmt.Fprintln(a.out, " p95 P95 usage for specific interface and period") + fmt.Fprintln(a.out, " alert Show/set alert thresholds") + fmt.Fprintln(a.out, " alert-routing Show/set alert delivery routing") + fmt.Fprintln(a.out, " alert-vm Show/set/check VM state alert rules") + fmt.Fprintln(a.out, " tag Add/remove/list cluster tags") + fmt.Fprintln(a.out, " kvm-tag Add/remove/list tags for specific KVM VM") + fmt.Fprintln(a.out, " drift Detect config/runtime drift") + fmt.Fprintln(a.out, " capacity Capacity forecast from disk usage history") + fmt.Fprintln(a.out, " slo Availability SLO report for cluster/VM") + fmt.Fprintln(a.out, " report Export cluster report (json/csv)") + fmt.Fprintln(a.out, " backup Archive selected paths and upload to SFTP/S3") + fmt.Fprintln(a.out, " repo-tunnel Configure temporary package repo access through a gateway VM") + fmt.Fprintln(a.out, " runbook List/show/run step-by-step scenarios") + fmt.Fprintln(a.out, " runbook-trigger Configure auto-triggered runbook on alerts") + fmt.Fprintln(a.out, " schedule Manage and run scheduled tasks") + fmt.Fprintln(a.out, " change-history Show applied change log") + fmt.Fprintln(a.out, " software Show/scan software plugin support (bird/frr/kvm/lxc/lxd)") + fmt.Fprintln(a.out, " disconnect Remove node") + fmt.Fprintln(a.out, " set-auth Change stored password or auth method (aliases: auth, password, passwd)") + fmt.Fprintln(a.out, " openssh Open interactive SSH shell to node (alias: ssh)") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Examples:") + fmt.Fprintln(a.out, " pxmon cluster connect --name eu-1 --host 10.0.0.10 --port 22 --user root --auth key --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/.ssh/id_ed25519.pxmonpassphrase") + fmt.Fprintln(a.out, " pxmon cluster connect --name vm-host-3 --host 198.51.100.30 --user root --type ipfabric --auth password --password 'example-pass' --store-password --insecure-host-key") + fmt.Fprintln(a.out, " pxmon cluster set-auth eu-1 --auth password --password 'example-pass' --store-password") + fmt.Fprintln(a.out, " pxmon cluster set-auth eu-1 --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/Desktop/passphrase.txt --store-key-passphrase") + fmt.Fprintln(a.out, " pxmon cluster set-auth vm-host-3 --type ipfabric") + fmt.Fprintln(a.out, " pxmon cluster bootstrap eu-1") + fmt.Fprintln(a.out, " pxmon cluster agent status") + fmt.Fprintln(a.out, " pxmon cluster agent update eu-1") + fmt.Fprintln(a.out, " pxmon cluster agent update eu-1 --restart-bot=true") + fmt.Fprintln(a.out, " pxmon cluster ping eu-1") + fmt.Fprintln(a.out, " pxmon cluster ping eu-1 --agent") + fmt.Fprintln(a.out, " pxmon cluster stats eu-1") + fmt.Fprintln(a.out, " pxmon cluster p95 eu-1 --iface eth0 --range 30d --graph") + fmt.Fprintln(a.out, " pxmon cluster alert set eu-1 --net-mbps 300 --ram 90 --disk 90") + fmt.Fprintln(a.out, " pxmon cluster alert set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup") + fmt.Fprintln(a.out, " pxmon cluster alert-routing set eu-1 --critical-immediate=true --warning-batch-mins 5") + fmt.Fprintln(a.out, " pxmon cluster alert-vm set eu-1 --enabled --warn-on-shutoff --min-running 100") + fmt.Fprintln(a.out, " pxmon cluster slo eu-1 --range 30d") + fmt.Fprintln(a.out, " pxmon cluster capacity forecast eu-1 --range 30d") + fmt.Fprintln(a.out, " pxmon cluster tag add eu-1 --tags prod,billing") + fmt.Fprintln(a.out, " pxmon cluster kvm-tag add eu-1 --vm vm123 --tags critical") + fmt.Fprintln(a.out, " pxmon cluster drift eu-1") + fmt.Fprintln(a.out, " pxmon cluster drift baseline set eu-1") + fmt.Fprintln(a.out, " pxmon cluster drift ack eu-1 --kind baseline_software --for 24h") + fmt.Fprintln(a.out, " pxmon cluster report export --format json --out ./cluster-report.json") + fmt.Fprintln(a.out, " pxmon cluster backup target add --name b2 --type s3 --s3-endpoint s3.example.net --s3-bucket backups --s3-access-key AKIA... --s3-secret-key ...") + fmt.Fprintln(a.out, " pxmon cluster backup target test b2") + fmt.Fprintln(a.out, " pxmon cluster backup plan add --name vm-images --cluster eu-1 --target b2 --path /var/lib/libvirt/images --every 6h") + fmt.Fprintln(a.out, " pxmon cluster backup run vm-images") + fmt.Fprintln(a.out, " pxmon cluster repo-tunnel install eu-1 --gateway 203.0.113.10:3128 --table 1010 -- dnf install -y curl jq") + fmt.Fprintln(a.out, " pxmon cluster runbook run vm-health-check") + fmt.Fprintln(a.out, " pxmon cluster runbook add --edit") + fmt.Fprintln(a.out, " pxmon cluster runbook-trigger set eu-1 --enabled --runbook-id vm-health-check --cooldown-mins 30") + fmt.Fprintln(a.out, " pxmon cluster runbook add --id custom-1 --name 'Custom' --step 'Check agent|cluster agent status' --step 'Drift|cluster drift'") + fmt.Fprintln(a.out, " pxmon cluster schedule add --edit") + fmt.Fprintln(a.out, " pxmon cluster schedule add --name audit --cmd 'cluster drift eu-1' --every 30m") + fmt.Fprintln(a.out, " pxmon cluster schedule add --name mk --cluster eu-1 --mode shell --cmd 'mkdir -p /tmp/test' --every 10m") + fmt.Fprintln(a.out, " pxmon cluster schedule start --interval 30s") + fmt.Fprintln(a.out, " pxmon cluster software scan eu-1") + fmt.Fprintln(a.out, " pxmon cluster openssh eu-1") +} + +func (a *App) runConfig(args []string, configPath string) int { + if len(args) == 0 { + a.printConfigHelp() + return 0 + } + sub := args[0] + rest := args[1:] + switch sub { + case "export": + return a.runConfigExport(rest, configPath) + case "import": + return a.runConfigImport(rest, configPath) + case "help", "--help", "-h": + a.printConfigHelp() + return 0 + default: + fmt.Fprintf(a.err, "unknown config command %q\n\n", sub) + a.printConfigHelp() + return 2 + } +} + +func (a *App) runConfigExport(args []string, configPath string) int { + fs := flag.NewFlagSet("config export", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Passphrase used to encrypt the export bundle") + out := fs.String("out", "", "Output file path (defaults to or pxmon-export.enc)") + if err := fs.Parse(args); err != nil { + return 2 + } + outPath := strings.TrimSpace(*out) + if outPath == "" && fs.NArg() > 0 { + outPath = strings.TrimSpace(fs.Arg(0)) + } + if outPath == "" { + outPath = "pxmon-export.enc" + } + pw := strings.TrimSpace(*password) + if pw == "" { + if !a.canPromptInteractive() { + fmt.Fprintln(a.err, "export: --password is required when running from TUI/non-interactive session") + return 2 + } + entered, err := promptPassword(a.out, "Export passphrase: ") + if err != nil { + fmt.Fprintf(a.err, "export: %v\n", err) + return 1 + } + confirm, err := promptPassword(a.out, "Confirm passphrase: ") + if err != nil { + fmt.Fprintf(a.err, "export: %v\n", err) + return 1 + } + if entered != confirm { + fmt.Fprintln(a.err, "export: passphrases do not match") + return 1 + } + pw = entered + } + if pw == "" { + fmt.Fprintln(a.err, "export: empty passphrase is not allowed") + return 1 + } + + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + + if err := svc.Export(outPath, pw); err != nil { + fmt.Fprintf(a.err, "export: %v\n", err) + return 1 + } + + abs, _ := filepath.Abs(outPath) + fmt.Fprintf(a.out, "%s %s\n", + colorLabel("Exported bundle:"), + colorAccent(abs), + ) + fmt.Fprintln(a.out, colorDim("keep the passphrase safe — without it the bundle cannot be restored")) + return 0 +} + +func (a *App) runConfigImport(args []string, configPath string) int { + fs := flag.NewFlagSet("config import", flag.ContinueOnError) + fs.SetOutput(a.err) + password := fs.String("password", "", "Passphrase used when the bundle was exported") + in := fs.String("in", "", "Input bundle path") + replace := fs.Bool("replace", false, "Wipe the local registry and replace it with the bundle") + if err := fs.Parse(args); err != nil { + return 2 + } + inPath := strings.TrimSpace(*in) + if inPath == "" && fs.NArg() > 0 { + inPath = strings.TrimSpace(fs.Arg(0)) + } + if inPath == "" { + fmt.Fprintln(a.err, "usage: pxmon config import [--password ] [--replace]") + return 2 + } + pw := strings.TrimSpace(*password) + if pw == "" { + if !a.canPromptInteractive() { + fmt.Fprintln(a.err, "import: --password is required when running from TUI/non-interactive session") + return 2 + } + entered, err := promptPassword(a.out, "Import passphrase: ") + if err != nil { + fmt.Fprintf(a.err, "import: %v\n", err) + return 1 + } + pw = entered + } + if pw == "" { + fmt.Fprintln(a.err, "import: empty passphrase is not allowed") + return 1 + } + + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + + mode := cluster.ImportModeMerge + if *replace { + mode = cluster.ImportModeReplace + } + + report, err := svc.Import(inPath, pw, mode) + if err != nil { + fmt.Fprintf(a.err, "import: %v\n", err) + return 1 + } + + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Import mode:"), colorValue(string(report.Mode))) + fmt.Fprintf(a.out, "%s %s %s %s %s %s\n", + colorLabel("added:"), colorValue(fmt.Sprintf("%d", report.Added)), + colorLabel("replaced:"), colorValue(fmt.Sprintf("%d", report.Replaced)), + colorLabel("total:"), colorValue(fmt.Sprintf("%d", report.TotalAfter)), + ) + if report.TelegramApplied { + fmt.Fprintln(a.out, colorLabel("telegram:")+" "+colorValue("applied")) + } + if report.LockerApplied { + fmt.Fprintln(a.out, colorLabel("locker:")+" "+colorValue("applied")) + } + return 0 +} + +func (a *App) printConfigHelp() { + fmt.Fprintln(a.out, "Usage: pxmon config [options]") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Commands:") + fmt.Fprintln(a.out, " export Save all clusters, settings and credentials to a passphrase-encrypted bundle") + fmt.Fprintln(a.out, " import Load a previously exported bundle (merge by default, --replace to wipe first)") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Examples:") + fmt.Fprintln(a.out, " pxmon config export ./backup.enc") + fmt.Fprintln(a.out, " pxmon config export --out ~/Desktop/pxmon.enc --password 'example-passphrase'") + fmt.Fprintln(a.out, " pxmon config import ./backup.enc") + fmt.Fprintln(a.out, " pxmon config import ./backup.enc --replace") +} + +// canPromptInteractive reports whether the current App instance can safely +// read a password from the user. When running inside the TUI console, the +// App is constructed with buffered out/err writers, so prompting would +// deadlock on os.Stdin that bubbletea already owns. +func (a *App) canPromptInteractive() bool { + if a == nil { + return false + } + if a.out != os.Stdout { + return false + } + return term.IsTerminal(int(os.Stdin.Fd())) +} + +func promptPassword(out io.Writer, prompt string) (string, error) { + fmt.Fprint(out, prompt) + stdinFd := int(os.Stdin.Fd()) + if term.IsTerminal(stdinFd) { + b, err := term.ReadPassword(stdinFd) + fmt.Fprintln(out) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil + } + // Non-interactive fallback: read a line from stdin. + buf := make([]byte, 4096) + n, err := os.Stdin.Read(buf) + if err != nil && n == 0 { + return "", err + } + return strings.TrimSpace(string(buf[:n])), nil +} + +func sanitizeCluster(c cluster.Cluster, revealSecrets bool) cluster.Cluster { + if revealSecrets { + return c + } + c.Password = maskSecret(c.Password) + c.KeyPassphrase = maskSecret(c.KeyPassphrase) + c.Agent.Token = maskSecret(c.Agent.Token) + c.Agent.RequestSecret = maskSecret(c.Agent.RequestSecret) + return c +} + +func printableSecret(v string) string { + if v == "" { + return "(empty)" + } + return v +} + +func emptyFallback(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +func writeJSON(w io.Writer, v any) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func maskSecret(secret string) string { + if secret == "" { + return "" + } + return "***" +} + +func ternary(cond bool, a, b string) string { + if cond { + return a + } + return b +} + +func splitLeadingSelector(args []string) (string, []string) { + if len(args) == 0 { + return "", args + } + first := strings.TrimSpace(args[0]) + if first == "" || strings.HasPrefix(first, "-") { + return "", args + } + return first, args[1:] +} + +func useANSIColor() bool { + if strings.TrimSpace(os.Getenv("NO_COLOR")) != "" { + return false + } + termName := strings.ToLower(strings.TrimSpace(os.Getenv("TERM"))) + if termName == "dumb" { + return false + } + return true +} + +const ( + ansiReset = "\x1b[0m" + ansiBold = "\x1b[1m" + ansiDim = "\x1b[2m" + ansiGreen = "\x1b[32m" + ansiYellow = "\x1b[33m" + ansiRed = "\x1b[31m" + ansiCyan = "\x1b[36m" + ansiBlue = "\x1b[34m" + ansiMagenta = "\x1b[35m" + ansiBrightBlack = "\x1b[90m" + ansiBrightRed = "\x1b[91m" + ansiBrightGreen = "\x1b[92m" + ansiBrightYellow = "\x1b[93m" + ansiBrightBlue = "\x1b[94m" + ansiBrightMagenta = "\x1b[95m" + ansiBrightCyan = "\x1b[96m" + ansiBrightWhite = "\x1b[97m" + ansiBoldCyan = "\x1b[1;36m" + ansiBoldBCyan = "\x1b[1;96m" + ansiBoldBMagenta = "\x1b[1;95m" + ansiBoldBGreen = "\x1b[1;92m" + ansiBoldBYellow = "\x1b[1;93m" + ansiBoldBRed = "\x1b[1;91m" + ansiBoldBBlue = "\x1b[1;94m" + ansiBoldWhite = "\x1b[1;97m" +) + +func colorWrap(code, v string) string { + if !useANSIColor() || v == "" { + return v + } + return code + v + ansiReset +} + +func colorOK(v string) string { return colorWrap(ansiBoldBGreen, v) } +func colorWarn(v string) string { return colorWrap(ansiBoldBYellow, v) } +func colorCrit(v string) string { return colorWrap(ansiBoldBRed, v) } +func colorHeader(v string) string { return colorWrap(ansiBoldBCyan, v) } +func colorLabel(v string) string { return colorWrap(ansiBoldCyan, v) } +func colorAccent(v string) string { return colorWrap(ansiBoldBMagenta, v) } +func colorInfo(v string) string { return colorWrap(ansiCyan, v) } +func colorBlue(v string) string { return colorWrap(ansiBrightBlue, v) } +func colorMagenta(v string) string { return colorWrap(ansiBrightMagenta, v) } +func colorDim(v string) string { return colorWrap(ansiDim, v) } +func colorMuted(v string) string { return colorWrap(ansiBrightBlack, v) } +func colorValue(v string) string { return colorWrap(ansiBoldWhite, v) } + +func shortVersion(v string) string { + s := strings.TrimSpace(v) + if s == "" { + return "unknown" + } + if len(s) > 12 { + return s[:12] + } + return s +} + +func fitCell(v string, width int) string { + if width <= 0 { + return "" + } + r := []rune(strings.TrimSpace(v)) + if len(r) > width { + if width <= 3 { + return string(r[:width]) + } + return string(r[:width-3]) + "..." + } + if len(r) < width { + return string(r) + strings.Repeat(" ", width-len(r)) + } + return string(r) +} + +func colorizeCell(v string, width int, color string) string { + padded := fitCell(v, width) + switch color { + case "ok": + return colorOK(padded) + case "warn": + return colorWarn(padded) + case "crit": + return colorCrit(padded) + case "header": + return colorHeader(padded) + case "label": + return colorLabel(padded) + case "accent": + return colorAccent(padded) + case "info": + return colorInfo(padded) + case "blue": + return colorBlue(padded) + case "magenta": + return colorMagenta(padded) + case "dim": + return colorDim(padded) + case "muted": + return colorMuted(padded) + case "value": + return colorValue(padded) + default: + return padded + } +} + +func clamp(v, minV, maxV int) int { + if v < minV { + return minV + } + if v > maxV { + return maxV + } + return v +} + +func shrinkOne(v *int, minV int) bool { + if v == nil || *v <= minV { + return false + } + *v = *v - 1 + return true +} + +func maxLen[T any](header string, pick func(T) string, rows []T) int { + maxV := len([]rune(strings.TrimSpace(header))) + for _, r := range rows { + n := len([]rune(strings.TrimSpace(pick(r)))) + if n > maxV { + maxV = n + } + } + return maxV +} + +func Main() { + app := New(os.Stdout, os.Stderr) + os.Exit(app.Run(os.Args[1:])) +} diff --git a/internal/cli/bot.go b/internal/cli/bot.go new file mode 100644 index 0000000..fd53cda --- /dev/null +++ b/internal/cli/bot.go @@ -0,0 +1,1523 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "os/signal" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +const ( + telegramBotDefaultPoll = 2 * time.Second + telegramBotRuntimeDir = "bot/telegram" + telegramBotPIDFile = "bot.pid" + telegramBotLogFile = "bot.log" +) + +func (a *App) runBot(args []string, configPath string, jsonOut bool) int { + if len(args) == 0 { + a.printBotHelp() + return 0 + } + + store, err := cluster.NewStore(configPath) + if err != nil { + fmt.Fprintf(a.err, "init config store: %v\n", err) + return 1 + } + svc := cluster.NewService(store) + svc.AttachNetworkStore(history.NewNetworkStore(svc.DataDir())) + + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "help", "-h", "--help": + a.printBotHelp() + return 0 + case "telegram", "tg": + return a.runBotTelegram(svc, configPath, args[1:], jsonOut) + default: + fmt.Fprintf(a.err, "unknown bot command %q\n\n", args[0]) + a.printBotHelp() + return 2 + } +} + +func (a *App) runBotTelegram(svc *cluster.Service, configPath string, args []string, jsonOut bool) int { + if len(args) == 0 { + a.printBotTelegramHelp() + return 0 + } + + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "help", "-h", "--help": + a.printBotTelegramHelp() + return 0 + case "show": + fs := flag.NewFlagSet("bot telegram show", flag.ContinueOnError) + fs.SetOutput(a.err) + showToken := fs.Bool("show-token", false, "Reveal token in plain text") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram show [--show-token]") + return 2 + } + + cfg, err := svc.GetTelegram() + if err != nil { + fmt.Fprintf(a.err, "telegram show: %v\n", err) + return 1 + } + if jsonOut { + out := cfg + if !*showToken { + out.Token = maskSecret(out.Token) + } + _ = writeJSON(a.out, out) + return 0 + } + + token := printableSecret(maskSecret(cfg.Token)) + if *showToken { + token = printableSecret(cfg.Token) + } + fmt.Fprintf(a.out, "Enabled: %t\n", cfg.Enabled) + fmt.Fprintf(a.out, "Token: %s\n", token) + fmt.Fprintf(a.out, "Allowed IDs: %s\n", formatInt64IDs(cfg.AllowedUserIDs)) + if !cfg.UpdatedAt.IsZero() { + fmt.Fprintf(a.out, "Updated: %s\n", cfg.UpdatedAt.Format(time.RFC3339)) + } + return 0 + + case "set": + fs := flag.NewFlagSet("bot telegram set", flag.ContinueOnError) + fs.SetOutput(a.err) + token := fs.String("token", "", "Telegram bot token") + enable := fs.Bool("enable", true, "Enable bot after updating config") + allowCSV := fs.String("allow-ids", "", "Comma-separated Telegram user IDs") + var allowList int64SliceFlag + fs.Var(&allowList, "allow", "Allowed Telegram user ID (repeatable)") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram set --token --allow [--allow ...]") + return 2 + } + + ids := append([]int64(nil), allowList...) + if strings.TrimSpace(*allowCSV) != "" { + fromCSV, err := parseInt64CSV(*allowCSV) + if err != nil { + fmt.Fprintf(a.err, "telegram set: %v\n", err) + return 2 + } + ids = append(ids, fromCSV...) + } + ids = normalizeInt64IDs(ids) + + if strings.TrimSpace(*token) == "" { + fmt.Fprintln(a.err, "telegram set: --token is required") + return 2 + } + if len(ids) == 0 { + fmt.Fprintln(a.err, "telegram set: at least one allowed telegram user id is required (--allow)") + return 2 + } + + updated, err := svc.SetTelegram(cluster.Telegram{ + Enabled: *enable, + Token: strings.TrimSpace(*token), + AllowedUserIDs: ids, + }) + if err != nil { + fmt.Fprintf(a.err, "telegram set: %v\n", err) + return 1 + } + if err := syncTelegramBotDaemon(svc, configPath, updated.Enabled, telegramBotDefaultPoll); err != nil { + fmt.Fprintf(a.err, "telegram set: %v\n", err) + return 1 + } + + if jsonOut { + out := updated + out.Token = maskSecret(out.Token) + _ = writeJSON(a.out, out) + return 0 + } + + fmt.Fprintln(a.out, "Telegram bot config updated.") + fmt.Fprintf(a.out, "Enabled: %t | Allowed IDs: %s\n", updated.Enabled, formatInt64IDs(updated.AllowedUserIDs)) + if updated.Enabled { + pidPath, logPath := telegramBotPaths(svc) + pid, running, _ := readTelegramBotPID(pidPath) + if running && pid > 0 { + fmt.Fprintf(a.out, "Background worker: running (pid %d)\n", pid) + } else { + fmt.Fprintln(a.out, "Background worker: starting") + } + fmt.Fprintf(a.out, "Logs: %s\n", logPath) + } + return 0 + + case "disable": + fs := flag.NewFlagSet("bot telegram disable", flag.ContinueOnError) + fs.SetOutput(a.err) + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram disable") + return 2 + } + + updated, err := svc.DisableTelegram() + if err != nil { + fmt.Fprintf(a.err, "telegram disable: %v\n", err) + return 1 + } + if err := syncTelegramBotDaemon(svc, configPath, false, telegramBotDefaultPoll); err != nil { + fmt.Fprintf(a.err, "telegram disable: %v\n", err) + return 1 + } + if jsonOut { + out := updated + out.Token = maskSecret(out.Token) + _ = writeJSON(a.out, out) + return 0 + } + fmt.Fprintf(a.out, "Telegram bot disabled. Allowed IDs: %s\n", formatInt64IDs(updated.AllowedUserIDs)) + _, logPath := telegramBotPaths(svc) + fmt.Fprintln(a.out, "Background worker: stopped") + fmt.Fprintf(a.out, "Logs: %s\n", logPath) + return 0 + + case "restart": + fs := flag.NewFlagSet("bot telegram restart", flag.ContinueOnError) + fs.SetOutput(a.err) + poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff for daemon run loop") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram restart [--poll 2s]") + return 2 + } + + cfg, err := svc.GetTelegram() + if err != nil { + fmt.Fprintf(a.err, "telegram restart: %v\n", err) + return 1 + } + if !cfg.Enabled { + _ = stopTelegramBotDaemon(svc, configPath) + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "enabled": false, + "running": false, + "reason": "telegram is disabled in settings", + }) + return 0 + } + fmt.Fprintln(a.out, "Telegram bot is disabled in settings; background worker remains stopped.") + return 0 + } + + pid, logPath, err := restartTelegramBotDaemon(svc, configPath, *poll) + if err != nil { + fmt.Fprintf(a.err, "telegram restart: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "enabled": true, + "running": true, + "pid": pid, + "log_path": logPath, + }) + return 0 + } + fmt.Fprintf(a.out, "Telegram bot restarted (pid %d).\n", pid) + fmt.Fprintf(a.out, "Logs: %s\n", logPath) + return 0 + + case "logs": + fs := flag.NewFlagSet("bot telegram logs", flag.ContinueOnError) + fs.SetOutput(a.err) + tail := fs.Int("tail", 200, "Number of last log lines") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram logs [--tail 200]") + return 2 + } + + cfg, cfgErr := svc.GetTelegram() + if cfgErr == nil && !cfg.Enabled { + _ = stopTelegramBotDaemon(svc, configPath) + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "running": false, + "reason": "telegram is disabled", + }) + return 0 + } + fmt.Fprintln(a.out, "Telegram process is not running (disabled in settings).") + return 0 + } + + pidPath, logPath := telegramBotPaths(svc) + pid, running, _ := readTelegramBotPID(pidPath) + lines, err := readLastLines(logPath, *tail) + if err != nil { + fmt.Fprintf(a.err, "telegram logs: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "running": running, + "pid": pid, + "pid_path": pidPath, + "log_path": logPath, + "lines": lines, + }) + return 0 + } + + fmt.Fprintf(a.out, "Running: %t", running) + if pid > 0 { + fmt.Fprintf(a.out, " (pid %d)", pid) + } + fmt.Fprintln(a.out) + if !running { + fmt.Fprintln(a.out, "Telegram process is not running.") + return 0 + } + fmt.Fprintf(a.out, "Log file: %s\n", logPath) + if len(lines) == 0 { + fmt.Fprintln(a.out, "(log is empty)") + return 0 + } + for _, line := range lines { + fmt.Fprintln(a.out, line) + } + return 0 + + case "run": + fs := flag.NewFlagSet("bot telegram run", flag.ContinueOnError) + fs.SetOutput(a.err) + poll := fs.Duration("poll", telegramBotDefaultPoll, "Polling backoff on errors") + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() > 0 { + fmt.Fprintln(a.err, "usage: pxmon bot telegram run [--poll 2s]") + return 2 + } + + cfg, err := svc.GetTelegram() + if err != nil { + fmt.Fprintf(a.err, "telegram run: %v\n", err) + return 1 + } + if strings.TrimSpace(cfg.Token) == "" { + fmt.Fprintln(a.err, "telegram run: bot token is not configured (use `pxmon bot telegram set ...`)") + return 1 + } + if len(cfg.AllowedUserIDs) == 0 { + fmt.Fprintln(a.err, "telegram run: no allowed users configured") + return 1 + } + if !cfg.Enabled { + fmt.Fprintln(a.err, "telegram run: bot is disabled, enable it with `pxmon bot telegram set --enable=true ...`") + return 1 + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + fmt.Fprintf(a.out, "Starting Telegram bot (allowed users: %s)\n", formatInt64IDs(cfg.AllowedUserIDs)) + err = runTelegramBotLoop(ctx, svc, configPath, cfg, *poll, a.err) + if err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintf(a.err, "telegram run: %v\n", err) + return 1 + } + fmt.Fprintln(a.out, "Telegram bot stopped.") + return 0 + + default: + fmt.Fprintf(a.err, "unknown telegram subcommand %q\n\n", args[0]) + a.printBotTelegramHelp() + return 2 + } +} + +func (a *App) printBotHelp() { + fmt.Fprintln(a.out, "pxmon bot commands:") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, " telegram Configure and run Telegram bot integration") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Run 'pxmon bot telegram --help' for details.") +} + +func (a *App) printBotTelegramHelp() { + fmt.Fprintln(a.out, "pxmon bot telegram commands:") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, " show Show Telegram bot config") + fmt.Fprintln(a.out, " set Set token and allowed Telegram user IDs") + fmt.Fprintln(a.out, " disable Disable Telegram bot") + fmt.Fprintln(a.out, " restart Restart Telegram bot background daemon") + fmt.Fprintln(a.out, " logs Show bot logs") + fmt.Fprintln(a.out, " run Start Telegram bot long-polling worker") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Examples:") + fmt.Fprintln(a.out, " pxmon bot telegram set --token 123:ABC --allow 111111111 --allow 222222222") + fmt.Fprintln(a.out, " pxmon bot telegram show") + fmt.Fprintln(a.out, " pxmon bot telegram restart") + fmt.Fprintln(a.out, " pxmon bot telegram logs --tail 100") + fmt.Fprintln(a.out, " pxmon bot telegram run") +} + +func syncTelegramBotDaemon(svc *cluster.Service, configPath string, enabled bool, poll time.Duration) error { + if enabled { + _, _, err := startTelegramBotDaemon(svc, configPath, poll) + return err + } + return stopTelegramBotDaemon(svc, configPath) +} + +func restartTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) { + if err := stopTelegramBotDaemon(svc, configPath); err != nil { + return 0, "", err + } + return startTelegramBotDaemon(svc, configPath, poll) +} + +func startTelegramBotDaemon(svc *cluster.Service, configPath string, poll time.Duration) (int, string, error) { + pidPath, logPath := telegramBotPaths(svc) + if err := os.MkdirAll(filepath.Dir(pidPath), 0o700); err != nil { + return 0, "", fmt.Errorf("create telegram runtime dir: %w", err) + } + + pid, running, err := readTelegramBotPID(pidPath) + if err == nil && running { + return pid, logPath, nil + } + if err == nil && !running { + _ = os.Remove(pidPath) + } + + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return 0, "", fmt.Errorf("open telegram log file: %w", err) + } + defer logFile.Close() + + exePath, err := os.Executable() + if err != nil { + return 0, "", fmt.Errorf("resolve executable path: %w", err) + } + + cfgPath := strings.TrimSpace(configPath) + if cfgPath == "" { + cfgPath = svc.ConfigPath() + } + + cmdArgs := make([]string, 0, 8) + if cfgPath != "" { + cmdArgs = append(cmdArgs, "--config", cfgPath) + } + cmdArgs = append(cmdArgs, "bot", "telegram", "run", "--poll", poll.String()) + + cmd := exec.Command(exePath, cmdArgs...) + cmd.Stdout = logFile + cmd.Stderr = logFile + cmd.Stdin = nil + cmd.Env = os.Environ() + + if err := cmd.Start(); err != nil { + return 0, "", fmt.Errorf("start telegram bot daemon: %w", err) + } + + pid = cmd.Process.Pid + if pid <= 0 { + return 0, "", errors.New("telegram bot daemon started with invalid pid") + } + _ = cmd.Process.Release() + if err := os.WriteFile(pidPath, []byte(strconv.Itoa(pid)+"\n"), 0o600); err != nil { + return 0, "", fmt.Errorf("write telegram pid file: %w", err) + } + return pid, logPath, nil +} + +func stopTelegramBotDaemon(svc *cluster.Service, configPath string) error { + pidPath, _ := telegramBotPaths(svc) + pid, running, err := readTelegramBotPID(pidPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + pid = 0 + } else { + return err + } + } + + cfgPath := strings.TrimSpace(configPath) + if cfgPath == "" && svc != nil { + cfgPath = strings.TrimSpace(svc.ConfigPath()) + } + + targets := map[int]struct{}{} + if pid > 0 { + targets[pid] = struct{}{} + } + for _, p := range discoverTelegramBotPIDs(cfgPath) { + if p > 0 { + targets[p] = struct{}{} + } + } + if len(targets) == 0 { + _ = os.Remove(pidPath) + return nil + } + if pid > 0 && !running { + delete(targets, pid) + } + + alive := make([]int, 0, len(targets)) + for p := range targets { + if processRunning(p) { + alive = append(alive, p) + } + } + if len(alive) == 0 { + _ = os.Remove(pidPath) + return nil + } + + for _, p := range alive { + _ = signalProcess(p, syscall.SIGTERM) + } + if waitForProcessesStop(alive, 4*time.Second) { + _ = os.Remove(pidPath) + return nil + } + + for _, p := range alive { + _ = signalProcess(p, syscall.SIGKILL) + } + if waitForProcessesStop(alive, 2*time.Second) { + _ = os.Remove(pidPath) + return nil + } + + still := make([]int, 0, len(alive)) + for _, p := range alive { + if processRunning(p) { + still = append(still, p) + } + } + if len(still) == 0 { + _ = os.Remove(pidPath) + return nil + } + return fmt.Errorf("telegram bot process(es) still running: %v", still) +} + +func signalProcess(pid int, sig syscall.Signal) error { + if pid <= 0 { + return nil + } + proc, err := os.FindProcess(pid) + if err != nil { + return nil + } + if err := proc.Signal(sig); err != nil { + if errors.Is(err, os.ErrProcessDone) { + return nil + } + // ESRCH: no such process + if strings.Contains(strings.ToLower(err.Error()), "no such process") { + return nil + } + return err + } + return nil +} + +func waitForProcessesStop(pids []int, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + allStopped := true + for _, p := range pids { + if processRunning(p) { + allStopped = false + break + } + } + if allStopped { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(120 * time.Millisecond) + } +} + +func discoverTelegramBotPIDs(configPath string) []int { + out, err := exec.Command("ps", "-axo", "pid=,command=").Output() + if err != nil { + return nil + } + + wantCfg := strings.TrimSpace(configPath) + lines := strings.Split(string(out), "\n") + seen := map[int]struct{}{} + pids := make([]int, 0, 8) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + pid, err := strconv.Atoi(fields[0]) + if err != nil || pid <= 0 { + continue + } + cmdline := strings.Join(fields[1:], " ") + if !strings.Contains(cmdline, "bot telegram run") { + continue + } + if !strings.Contains(cmdline, "pxmon") { + continue + } + if wantCfg != "" && !strings.Contains(cmdline, wantCfg) { + continue + } + if _, ok := seen[pid]; ok { + continue + } + seen[pid] = struct{}{} + pids = append(pids, pid) + } + return pids +} + +func telegramBotPaths(svc *cluster.Service) (string, string) { + baseDir := "." + if svc != nil { + baseDir = svc.DataDir() + } + runtimeDir := filepath.Join(baseDir, telegramBotRuntimeDir) + return filepath.Join(runtimeDir, telegramBotPIDFile), filepath.Join(runtimeDir, telegramBotLogFile) +} + +func readTelegramBotPID(pidPath string) (int, bool, error) { + raw, err := os.ReadFile(pidPath) + if err != nil { + return 0, false, err + } + text := strings.TrimSpace(string(raw)) + if text == "" { + return 0, false, fmt.Errorf("empty pid file: %s", pidPath) + } + pid, err := strconv.Atoi(text) + if err != nil || pid <= 0 { + return 0, false, fmt.Errorf("invalid pid in %s", pidPath) + } + return pid, processRunning(pid), nil +} + +func processRunning(pid int) bool { + if pid <= 0 { + return false + } + out, err := exec.Command("ps", "-o", "stat=", "-p", strconv.Itoa(pid)).Output() + if err != nil { + return false + } + state := strings.TrimSpace(string(out)) + if state == "" { + return false + } + // Zombie should be treated as not running for lifecycle control. + if strings.HasPrefix(state, "Z") { + return false + } + return true +} + +func readLastLines(path string, tail int) ([]string, error) { + if tail <= 0 { + tail = 200 + } + + f, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + return nil, err + } + defer f.Close() + + lines := make([]string, 0, tail) + buf := make([]string, tail) + idx := 0 + total := 0 + + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for sc.Scan() { + line := strings.TrimRight(sc.Text(), "\r") + buf[idx%tail] = line + idx++ + total++ + } + if err := sc.Err(); err != nil { + return nil, err + } + + if total == 0 { + return lines, nil + } + + start := 0 + if total > tail { + start = idx % tail + total = tail + } + lines = make([]string, 0, total) + for i := 0; i < total; i++ { + lines = append(lines, buf[(start+i)%tail]) + } + return lines, nil +} + +func runTelegramBotLoop(ctx context.Context, svc *cluster.Service, configPath string, cfg cluster.Telegram, pollBackoff time.Duration, logErr io.Writer) error { + if pollBackoff < 500*time.Millisecond { + pollBackoff = 500 * time.Millisecond + } + + client := &http.Client{ + Timeout: 70 * time.Second, + } + + // Background sampler: appends a network history snapshot for every + // cluster with an agent every 30s so P95/graph commands have data to + // work with even when the TUI isn't running. + samplerCtx, cancelSampler := context.WithCancel(ctx) + defer cancelSampler() + go runHistorySampler(samplerCtx, svc, 30*time.Second, logErr) + + var offset int64 + lastVMAlertScan := time.Time{} + lastNetSustainAlertScan := time.Time{} + for { + liveCfg, err := svc.GetTelegram() + if err == nil { + if !liveCfg.Enabled { + fmt.Fprintln(logErr, "telegram bot disabled in config, stopping worker") + return nil + } + if strings.TrimSpace(liveCfg.Token) == "" { + fmt.Fprintln(logErr, "telegram bot token is empty in config, stopping worker") + return nil + } + if len(liveCfg.AllowedUserIDs) == 0 { + fmt.Fprintln(logErr, "telegram bot allowed_user_ids is empty in config, stopping worker") + return nil + } + cfg = liveCfg + } + + allowed := make(map[int64]struct{}, len(cfg.AllowedUserIDs)) + for _, id := range cfg.AllowedUserIDs { + allowed[id] = struct{}{} + } + if time.Since(lastVMAlertScan) > 60*time.Second { + lastVMAlertScan = time.Now() + pollAndSendVMAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr) + } + if time.Since(lastNetSustainAlertScan) > 60*time.Second { + lastNetSustainAlertScan = time.Now() + pollAndSendSustainedNetAlerts(ctx, client, svc, cfg.Token, cfg.AllowedUserIDs, logErr) + } + + updates, nextOffset, err := telegramGetUpdates(ctx, client, cfg.Token, offset, 50) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + fmt.Fprintf(logErr, "telegram poll failed: %v\n", err) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(pollBackoff): + } + continue + } + offset = nextOffset + + for _, upd := range updates { + if upd.Message == nil { + continue + } + msg := upd.Message + text := strings.TrimSpace(msg.Text) + if text == "" { + continue + } + + userID := int64(0) + if msg.From != nil { + userID = msg.From.ID + } + if _, ok := allowed[userID]; !ok { + _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotAccessDeniedHTML(userID)) + continue + } + + cmd := normalizeTelegramCommand(text) + if cmd == "" { + continue + } + + if cmd == "help" { + _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, formatBotHelpHTML()) + continue + } + + if handled := handleBotUsageCommand(ctx, client, svc, cfg.Token, msg.Chat.ID, cmd); handled { + continue + } + + out, code := runObserverScopedCommand(svc, configPath, cmd, observerCommandOptions{ + AllowShellEscape: false, + StatsAutoOnce: true, + BlockBotRun: true, + StripANSI: true, + }) + if code == 0 { + if sent, sendErr := trySendGraphAttachmentFromCLIOutput(ctx, client, cfg.Token, msg.Chat.ID, cmd, out); sendErr != nil { + _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, + fmt.Sprintf("🔴 graph\n
send failed: %s
", htmlEscapeTelegram(sendErr.Error()))) + continue + } else if sent { + continue + } + } + for _, chunk := range formatBotCommandReplyHTML(cmd, out, code) { + _ = telegramSendHTML(ctx, client, cfg.Token, msg.Chat.ID, chunk) + } + } + } +} + +type tgUpdatesResponse struct { + OK bool `json:"ok"` + Result []tgUpdate `json:"result"` + Description string `json:"description"` +} + +type tgUpdate struct { + UpdateID int64 `json:"update_id"` + Message *tgMessage `json:"message"` +} + +type tgMessage struct { + MessageID int64 `json:"message_id"` + From *tgUser `json:"from"` + Chat tgChat `json:"chat"` + Text string `json:"text"` +} + +type tgUser struct { + ID int64 `json:"id"` + Username string `json:"username"` +} + +type tgChat struct { + ID int64 `json:"id"` +} + +var botVMAlertState = struct { + sync.Mutex + last map[string]string + lastWarnSent map[string]time.Time +}{ + last: map[string]string{}, + lastWarnSent: map[string]time.Time{}, +} + +var botNetSustainAlertState = struct { + sync.Mutex + active map[string]bool + lastSent map[string]time.Time +}{ + active: map[string]bool{}, + lastSent: map[string]time.Time{}, +} + +func pollAndSendSustainedNetAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) { + store := svc.NetworkStore() + if store == nil { + return + } + clusters, _, err := svc.List() + if err != nil { + return + } + now := time.Now().UTC() + + for _, c := range clusters { + policy, err := svc.GetAlertPolicy(c.ID) + if err != nil || !policy.NetSustainEnabled || policy.NetSustainMbps <= 0 { + continue + } + + windowMins := policy.NetSustainMinutes + if windowMins <= 0 { + windowMins = 60 + } + cooldownMins := policy.NetSustainCooldownMins + if cooldownMins <= 0 { + cooldownMins = 30 + } + window := time.Duration(windowMins) * time.Minute + cooldown := time.Duration(cooldownMins) * time.Minute + + snaps, err := store.Load(c.ID, now.Add(-window)) + if err != nil { + fmt.Fprintf(logErr, "net-alert worker: %s: %v\n", c.Name, err) + continue + } + if len(snaps) == 0 { + botNetSustainAlertState.Lock() + delete(botNetSustainAlertState.active, c.ID) + botNetSustainAlertState.Unlock() + continue + } + + ifaces := sustainedCandidateIfaces(snaps, policy) + if len(ifaces) == 0 { + continue + } + + seenKeys := make(map[string]struct{}, len(ifaces)) + minCoverage := time.Duration(float64(window) * 0.9) + for _, iface := range ifaces { + key := c.ID + "|" + iface + seenKeys[key] = struct{}{} + + series := history.AggregateNodeSeries(snaps, iface) + if len(series) == 0 { + botNetSustainAlertState.Lock() + delete(botNetSustainAlertState.active, key) + delete(botNetSustainAlertState.lastSent, key) + botNetSustainAlertState.Unlock() + continue + } + + coverage := series[len(series)-1].Timestamp.Sub(series[0].Timestamp) + if coverage < minCoverage { + continue + } + + minV := series[0].TotalMbps + maxV := series[0].TotalMbps + sumV := 0.0 + triggered := true + for _, p := range series { + v := p.TotalMbps + sumV += v + if v < minV { + minV = v + } + if v > maxV { + maxV = v + } + if v < policy.NetSustainMbps { + triggered = false + } + } + + botNetSustainAlertState.Lock() + prevActive := botNetSustainAlertState.active[key] + lastSent := botNetSustainAlertState.lastSent[key] + if !triggered { + botNetSustainAlertState.active[key] = false + botNetSustainAlertState.Unlock() + continue + } + if prevActive && !lastSent.IsZero() && now.Sub(lastSent) < cooldown { + botNetSustainAlertState.Unlock() + continue + } + botNetSustainAlertState.active[key] = true + botNetSustainAlertState.lastSent[key] = now + botNetSustainAlertState.Unlock() + + avgV := sumV / float64(len(series)) + p95 := history.PercentileMbps(series, 95) + msg := "🔴 Sustained network alert (CRITICAL)\n" + + "Cluster: " + htmlEscapeTelegram(c.Name) + "\n" + + "Interface: " + htmlEscapeTelegram(iface) + "\n" + + fmt.Sprintf("Condition: ≥ %.1f Mbps for %d min\n", policy.NetSustainMbps, windowMins) + + fmt.Sprintf("Observed: min %.1f · avg %.1f · p95 %.1f · max %.1f Mbps (%d samples)", + minV, avgV, p95, maxV, len(series)) + for _, id := range userIDs { + _ = telegramSendHTML(ctx, client, token, id, msg) + } + } + + botNetSustainAlertState.Lock() + for k := range botNetSustainAlertState.active { + if !strings.HasPrefix(k, c.ID+"|") { + continue + } + if _, ok := seenKeys[k]; ok { + continue + } + delete(botNetSustainAlertState.active, k) + delete(botNetSustainAlertState.lastSent, k) + } + botNetSustainAlertState.Unlock() + } +} + +func sustainedCandidateIfaces(snaps []history.NetworkSnapshot, policy cluster.AlertPolicy) []string { + seen := map[string]struct{}{} + out := make([]string, 0, 16) + pinned := strings.TrimSpace(policy.NetSustainIface) + if pinned != "" { + return []string{pinned} + } + for _, snap := range snaps { + for _, s := range snap.Interfaces { + iface := strings.TrimSpace(s.Interface) + if iface == "" { + continue + } + if _, ok := seen[iface]; ok { + continue + } + if !ifaceAllowedByFilters(iface, policy.NetSustainInclude, policy.NetSustainExclude) { + continue + } + seen[iface] = struct{}{} + out = append(out, iface) + } + } + sort.Strings(out) + return out +} + +func ifaceAllowedByFilters(iface string, include, exclude []string) bool { + n := strings.ToLower(strings.TrimSpace(iface)) + if n == "" { + return false + } + if len(include) > 0 { + matched := false + for _, tok := range include { + t := strings.ToLower(strings.TrimSpace(tok)) + if t == "" { + continue + } + if strings.Contains(n, t) { + matched = true + break + } + } + if !matched { + return false + } + } + for _, tok := range exclude { + t := strings.ToLower(strings.TrimSpace(tok)) + if t == "" { + continue + } + if strings.Contains(n, t) { + return false + } + } + return true +} + +func telegramGetUpdates(ctx context.Context, client *http.Client, token string, offset int64, timeoutSeconds int) ([]tgUpdate, int64, error) { + if timeoutSeconds < 5 { + timeoutSeconds = 5 + } + + values := url.Values{} + if offset > 0 { + values.Set("offset", strconv.FormatInt(offset, 10)) + } + values.Set("timeout", strconv.Itoa(timeoutSeconds)) + values.Set("allowed_updates", `["message"]`) + + endpoint := "https://api.telegram.org/bot" + token + "/getUpdates?" + values.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return nil, offset, err + } + + resp, err := client.Do(req) + if err != nil { + return nil, offset, err + } + defer resp.Body.Close() + + var payload tgUpdatesResponse + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return nil, offset, err + } + if !payload.OK { + msg := strings.TrimSpace(payload.Description) + if msg == "" { + msg = "telegram getUpdates returned ok=false" + } + return nil, offset, errors.New(msg) + } + + next := offset + for _, upd := range payload.Result { + if upd.UpdateID >= next { + next = upd.UpdateID + 1 + } + } + return payload.Result, next, nil +} + +func telegramSendText(ctx context.Context, client *http.Client, token string, chatID int64, text string) error { + text = strings.TrimSpace(text) + if text == "" { + return nil + } + + chunks := splitTelegramText(text, 3500) + for _, chunk := range chunks { + values := url.Values{} + values.Set("chat_id", strconv.FormatInt(chatID, 10)) + values.Set("text", chunk) + + endpoint := "https://api.telegram.org/bot" + token + "/sendMessage" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode())) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := client.Do(req) + if err != nil { + return err + } + _ = resp.Body.Close() + } + return nil +} + +func telegramSendHTML(ctx context.Context, client *http.Client, token string, chatID int64, html string) error { + html = strings.TrimSpace(html) + if html == "" { + return nil + } + values := url.Values{} + values.Set("chat_id", strconv.FormatInt(chatID, 10)) + values.Set("text", html) + values.Set("parse_mode", "HTML") + values.Set("disable_web_page_preview", "true") + + endpoint := "https://api.telegram.org/bot" + token + "/sendMessage" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(values.Encode())) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := client.Do(req) + if err != nil { + return err + } + _ = resp.Body.Close() + return nil +} + +func htmlEscapeTelegram(s string) string { + s = strings.ReplaceAll(s, "&", "&") + s = strings.ReplaceAll(s, "<", "<") + s = strings.ReplaceAll(s, ">", ">") + return s +} + +func formatBotHelpHTML() string { + return strings.Join([]string{ + "✻ PXmon (Phylex Monitor) Bot", + "Secure remote control for your cluster fleet.", + "", + "━━ Clusters", + "• cluster list", + "• cluster show eu-1", + "• cluster ping eu-1 --agent", + "• cluster stats eu-1", + "", + "━━ Usage & billing", + "• usage eu-1 — live processes, RAM, folders, top iface", + "• traffic eu-1 1h|1d|1mo|all — P95 text summary", + "• graph eu-1 1d — PNG network graph with P95 line", + "• p95 eu-1 eth0 30d — interface P95 + graph attachment", + "", + "━━ Alerts", + "• alerts set eu-1 --net-mbps 300 --ram 90 --disk 90", + "• alerts set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup", + "• alert-vm check eu-1", + "", + "━━ Virtualization", + "• kvm list --cluster eu-1", + "• lxd top --cluster eu-1", + "", + "━━ Shortcuts", + "/help · /start", + }, "\n") +} + +func formatBotAccessDeniedHTML(userID int64) string { + idRepr := "unknown" + if userID > 0 { + idRepr = strconv.FormatInt(userID, 10) + } + return strings.Join([]string{ + "🔒 Access denied", + fmt.Sprintf("Your Telegram ID %s is not on the allow-list.", idRepr), + "", + "Ask the cluster administrator to add it via", + "pxmon bot telegram allow <id>", + }, "\n") +} + +func pollAndSendVMAlerts(ctx context.Context, client *http.Client, svc *cluster.Service, token string, userIDs []int64, logErr io.Writer) { + clusters, _, err := svc.List() + if err != nil { + return + } + for _, c := range clusters { + p, err := svc.GetVMAlertPolicy(c.ID) + if err != nil || !p.Enabled { + continue + } + routing, _ := svc.GetAlertRouting(c.ID) + trigger, _ := svc.GetRunbookTrigger(c.ID) + callCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + rep, err := svc.CheckVMAlerts(callCtx, c.ID) + cancel() + if err != nil { + fmt.Fprintf(logErr, "vm-alert worker: %s: %v\n", c.Name, err) + continue + } + state := strings.Join(rep.Warnings, "|") + botVMAlertState.Lock() + prev := botVMAlertState.last[c.ID] + lastWarn := botVMAlertState.lastWarnSent[c.ID] + if state == "" { + delete(botVMAlertState.last, c.ID) + delete(botVMAlertState.lastWarnSent, c.ID) + } else { + botVMAlertState.last[c.ID] = state + } + botVMAlertState.Unlock() + if state == "" || state == prev { + continue + } + isCritical := false + for _, w := range rep.Warnings { + if strings.Contains(strings.ToLower(w), "min_running") || strings.Contains(strings.ToLower(w), "below") { + isCritical = true + break + } + } + if !isCritical { + batchDur := time.Duration(routing.WarningBatchMins) * time.Minute + if batchDur <= 0 { + batchDur = 5 * time.Minute + } + if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur { + continue + } + botVMAlertState.Lock() + botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC() + botVMAlertState.Unlock() + } else if !routing.CriticalImmediate { + // If critical-immediate is disabled, still route as warning batch. + batchDur := time.Duration(routing.WarningBatchMins) * time.Minute + if batchDur <= 0 { + batchDur = 5 * time.Minute + } + if !lastWarn.IsZero() && time.Since(lastWarn) < batchDur { + continue + } + botVMAlertState.Lock() + botVMAlertState.lastWarnSent[c.ID] = time.Now().UTC() + botVMAlertState.Unlock() + } + msg := "⚠️ VM alerts\n" + + "Cluster: " + htmlEscapeTelegram(c.Name) + "\n" + + fmt.Sprintf("Running: %d Shut off: %d\n", rep.Running, rep.ShutOff) + if len(rep.ShutOffNames) > 0 { + msg += "Shut off VM: " + htmlEscapeTelegram(strings.Join(rep.ShutOffNames, ", ")) + "\n" + } + for _, w := range rep.Warnings { + msg += "• " + htmlEscapeTelegram(w) + "\n" + } + for _, id := range userIDs { + _ = telegramSendHTML(ctx, client, token, id, msg) + } + // Auto-trigger runbook on VM shutoff alert. + if trigger.Enabled && trigger.OnVMShutoff && rep.ShutOff > 0 && strings.TrimSpace(trigger.RunbookID) != "" { + cooldown := time.Duration(trigger.CooldownMins) * time.Minute + if cooldown <= 0 { + cooldown = 30 * time.Minute + } + if trigger.LastTriggered.IsZero() || time.Since(trigger.LastTriggered) >= cooldown { + if err := executeRunbookByID(svc, trigger.RunbookID); err != nil { + fmt.Fprintf(logErr, "runbook trigger: %s: %v\n", c.Name, err) + } else { + _ = svc.TouchRunbookTrigger(c.ID, time.Now().UTC()) + for _, id := range userIDs { + _ = telegramSendHTML(ctx, client, token, id, + "🟠 runbook auto-triggered\n"+ + "Cluster: "+htmlEscapeTelegram(c.Name)+"\n"+ + "Runbook: "+htmlEscapeTelegram(trigger.RunbookID)) + } + } + } + } + } +} + +func executeRunbookByID(svc *cluster.Service, runbookID string) error { + rb, ok := svc.GetRunbook(runbookID) + if !ok { + return errors.New("runbook not found: " + runbookID) + } + for _, st := range rb.Steps { + cmd := strings.TrimSpace(st.Command) + if cmd == "" { + continue + } + _, code := runObserverScopedCommand(svc, svc.ConfigPath(), cmd, observerCommandOptions{ + AllowShellEscape: false, + StatsAutoOnce: true, + BlockBotRun: true, + StripANSI: true, + }) + if code != 0 { + return errors.New("step failed: " + st.Title) + } + } + return nil +} + +func formatBotCommandReplyHTML(cmd, out string, code int) []string { + cmd = strings.TrimSpace(cmd) + body := strings.TrimRight(out, "\n") + + var header string + if code == 0 { + header = fmt.Sprintf("🟢 %s", htmlEscapeTelegram(cmd)) + } else { + header = fmt.Sprintf("🔴 %s exit %d", htmlEscapeTelegram(cmd), code) + } + + if strings.TrimSpace(body) == "" { + if code == 0 { + return []string{header + "\n
ok
"} + } + return []string{header + "\n
(no output)
"} + } + + bodyChunks := splitTelegramText(htmlEscapeTelegram(body), 3600) + msgs := make([]string, 0, len(bodyChunks)) + for i, chunk := range bodyChunks { + if i == 0 { + msgs = append(msgs, header+"\n
"+chunk+"
") + } else { + msgs = append(msgs, "
"+chunk+"
") + } + } + return msgs +} + +func splitTelegramText(text string, limit int) []string { + if limit < 256 { + limit = 256 + } + if len(text) <= limit { + return []string{text} + } + lines := strings.Split(text, "\n") + out := make([]string, 0, len(lines)) + var cur strings.Builder + for _, line := range lines { + candidate := line + if cur.Len() > 0 { + candidate = "\n" + line + } + if cur.Len()+len(candidate) > limit { + if cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + } + for len(line) > limit { + out = append(out, line[:limit]) + line = line[limit:] + } + if line != "" { + cur.WriteString(line) + } + continue + } + cur.WriteString(candidate) + } + if cur.Len() > 0 { + out = append(out, cur.String()) + } + return out +} + +func normalizeTelegramCommand(text string) string { + text = strings.TrimSpace(text) + if text == "" { + return "" + } + if strings.HasPrefix(text, "/") { + text = strings.TrimPrefix(text, "/") + parts := strings.Fields(text) + if len(parts) == 0 { + return "" + } + parts[0] = strings.Split(parts[0], "@")[0] + text = strings.Join(parts, " ") + } + if strings.EqualFold(text, "start") { + return "help" + } + if strings.HasPrefix(strings.ToLower(text), "run ") { + return strings.TrimSpace(text[4:]) + } + return text +} + +func parseInt64CSV(raw string) ([]int64, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + parts := strings.Split(raw, ",") + out := make([]int64, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + v, err := strconv.ParseInt(p, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid telegram user id %q", p) + } + out = append(out, v) + } + return out, nil +} + +func formatInt64IDs(ids []int64) string { + if len(ids) == 0 { + return "(none)" + } + c := append([]int64(nil), ids...) + sort.Slice(c, func(i, j int) bool { return c[i] < c[j] }) + parts := make([]string, 0, len(c)) + for _, id := range c { + parts = append(parts, strconv.FormatInt(id, 10)) + } + return strings.Join(parts, ",") +} + +func normalizeInt64IDs(ids []int64) []int64 { + if len(ids) == 0 { + return nil + } + seen := make(map[int64]struct{}, len(ids)) + out := make([]int64, 0, len(ids)) + for _, id := range ids { + if id <= 0 { + continue + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + sort.Slice(out, func(i, j int) bool { return out[i] < out[j] }) + if len(out) == 0 { + return nil + } + return out +} + +type int64SliceFlag []int64 + +func (s *int64SliceFlag) String() string { + return formatInt64IDs(*s) +} + +func (s *int64SliceFlag) Set(v string) error { + id, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64) + if err != nil { + return fmt.Errorf("invalid user id %q", v) + } + *s = append(*s, id) + return nil +} diff --git a/internal/cli/bot_usage.go b/internal/cli/bot_usage.go new file mode 100644 index 0000000..d0a6da0 --- /dev/null +++ b/internal/cli/bot_usage.go @@ -0,0 +1,440 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +type samplerIfaceCounter struct { + RxBytes uint64 + TxBytes uint64 +} + +type samplerState struct { + Timestamp time.Time + Counters map[string]samplerIfaceCounter +} + +var ( + samplerMu sync.Mutex + samplerPrev = map[string]*samplerState{} +) + +// handleBotUsageCommand intercepts usage/traffic/graph commands before they +// reach the generic runObserverScopedCommand bridge so it can send richer +// payloads (including PNG attachments) back to Telegram. Returns true when +// the command was consumed. +func handleBotUsageCommand(ctx context.Context, client *http.Client, svc *cluster.Service, token string, chatID int64, cmd string) bool { + fields := strings.Fields(cmd) + if len(fields) == 0 { + return false + } + rest := fields + if strings.EqualFold(fields[0], "cluster") && len(fields) > 1 { + rest = fields[1:] + } + head := strings.ToLower(rest[0]) + if head != "usage" && head != "traffic" && head != "graph" && head != "p95" { + return false + } + + selector := "" + rangeStr := defaultBotRange(head) + iface := "" + for _, f := range rest[1:] { + if strings.HasPrefix(f, "--iface=") { + iface = strings.TrimSpace(strings.TrimPrefix(f, "--iface=")) + continue + } + if strings.HasPrefix(f, "--range=") { + rangeStr = strings.TrimPrefix(f, "--range=") + continue + } + if rng, ok := history.ParseRangeShortcut(f); ok { + rangeStr = string(rng) + continue + } + if selector == "" { + if head == "p95" && iface == "" && !strings.HasPrefix(f, "-") { + iface = f + } else { + selector = f + } + } + } + + rng, ok := history.ParseRangeShortcut(strings.TrimSpace(rangeStr)) + if !ok { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 %s\n
invalid range %q
", head, rangeStr)) + return true + } + + gather, cancel := context.WithTimeout(ctx, 50*time.Second) + defer cancel() + + snap, err := svc.CollectUsageSnapshot(gather, selector, rng, "/") + if err != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 %s\n
%s
", head, htmlEscapeTelegram(err.Error()))) + return true + } + + switch head { + case "usage": + body := formatUsageForBot(snap, rng) + header := fmt.Sprintf("🟢 usage %s %s", htmlEscapeTelegram(snap.ClusterName), rng.Label()) + for _, chunk := range splitTelegramText(htmlEscapeTelegram(body), 3400) { + _ = telegramSendHTML(ctx, client, token, chatID, header+"\n
"+chunk+"
") + header = "" + } + case "traffic": + body := formatTrafficForBot(snap, rng) + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🟢 traffic %s %s\n
%s
", + htmlEscapeTelegram(snap.ClusterName), rng.Label(), htmlEscapeTelegram(body))) + case "graph": + png, err := cluster.RenderUsageChartPNG(snap, "") + if err != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 graph\n
%s
", htmlEscapeTelegram(err.Error()))) + return true + } + caption := fmt.Sprintf("📈 %s %s\nP95: %s · max: %s · avg: %s · %d samples", + htmlEscapeTelegram(snap.ClusterName), + rng.Label(), + formatMbpsHuman(snap.P95TotalMbps), + formatMbpsHuman(snap.MaxTotalMbps), + formatMbpsHuman(snap.AvgTotalMbps), + len(snap.NodeSeries), + ) + filename := sanitizeFilename(snap.ClusterName) + "-" + string(rng) + ".png" + if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 graph\n
send failed: %s
", htmlEscapeTelegram(sendErr.Error()))) + } + case "p95": + if strings.TrimSpace(iface) == "" { + _ = telegramSendHTML(ctx, client, token, chatID, "🔴 p95\n
iface is required (example: p95 eth0 30d)
") + return true + } + p95Snap, err := svc.CollectInterfaceP95(selector, iface, rng) + if err != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 p95\n
%s
", htmlEscapeTelegram(err.Error()))) + return true + } + png, err := svc.RenderInterfaceP95GraphPNG(p95Snap) + if err != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 p95\n
render failed: %s
", htmlEscapeTelegram(err.Error()))) + return true + } + filename := sanitizeFilename(p95Snap.ClusterName) + "-" + sanitizeFilename(p95Snap.Interface) + "-" + string(rng) + ".png" + caption := fmt.Sprintf("📈 P95 %s %s\niface: %s\nP95: %s · max: %s · avg: %s · %d samples", + htmlEscapeTelegram(p95Snap.ClusterName), + rng.Label(), + htmlEscapeTelegram(p95Snap.Interface), + formatMbpsHuman(p95Snap.P95Mbps), + formatMbpsHuman(p95Snap.MaxMbps), + formatMbpsHuman(p95Snap.AvgMbps), + p95Snap.Samples, + ) + if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil { + _ = telegramSendHTML(ctx, client, token, chatID, + fmt.Sprintf("🔴 p95\n
send failed: %s
", htmlEscapeTelegram(sendErr.Error()))) + } + } + return true +} + +func defaultBotRange(cmd string) string { + switch cmd { + case "usage": + return "live" + case "traffic": + return "1h" + case "graph": + return "1d" + case "p95": + return "30d" + } + return "live" +} + +// trySendGraphAttachmentFromCLIOutput is a safety net for graph commands that +// were executed through the generic CLI bridge and returned text like: +// +// Saved: +// +// It uploads the generated PNG so Telegram users always receive the image. +func trySendGraphAttachmentFromCLIOutput(ctx context.Context, client *http.Client, token string, chatID int64, cmd, out string) (bool, error) { + fields := strings.Fields(strings.TrimSpace(cmd)) + if len(fields) == 0 { + return false, nil + } + + isGraph := false + if strings.EqualFold(fields[0], "graph") { + isGraph = true + } + if len(fields) >= 2 && strings.EqualFold(fields[0], "cluster") && strings.EqualFold(fields[1], "graph") { + isGraph = true + } + if !isGraph { + return false, nil + } + + lines := strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n") + saved := "" + p95 := "" + for _, line := range lines { + t := strings.TrimSpace(stripANSI(line)) + if strings.HasPrefix(strings.ToLower(t), "saved:") { + saved = strings.TrimSpace(t[len("saved:"):]) + continue + } + if strings.HasPrefix(strings.ToLower(t), "p95:") { + p95 = t + } + } + if saved == "" { + return false, nil + } + + data, err := os.ReadFile(saved) + if err != nil { + return false, fmt.Errorf("read graph png %q: %w", saved, err) + } + + filename := filepath.Base(saved) + caption := "📈 cluster graph" + if p95 != "" { + caption += "\n" + htmlEscapeTelegram(p95) + } + if err := telegramSendPhoto(ctx, client, token, chatID, filename, data, caption); err != nil { + return false, err + } + return true, nil +} + +func formatTrafficForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string { + var b strings.Builder + fmt.Fprintf(&b, "range: %s\n", rng.Label()) + fmt.Fprintf(&b, "samples: %d\n", len(snap.NodeSeries)) + fmt.Fprintf(&b, "P95: %s\n", formatMbpsHuman(snap.P95TotalMbps)) + fmt.Fprintf(&b, "max: %s\n", formatMbpsHuman(snap.MaxTotalMbps)) + fmt.Fprintf(&b, "avg: %s\n", formatMbpsHuman(snap.AvgTotalMbps)) + if snap.TopIfaceName != "" { + fmt.Fprintf(&b, "uplink: %s (avg %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps)) + } + if snap.HistoryError != "" { + fmt.Fprintf(&b, "note: %s\n", snap.HistoryError) + } + return b.String() +} + +// telegramSendPhoto uploads a PNG to Telegram via multipart sendPhoto. +func telegramSendPhoto(ctx context.Context, client *http.Client, token string, chatID int64, filename string, png []byte, captionHTML string) error { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + if err := w.WriteField("chat_id", strconv.FormatInt(chatID, 10)); err != nil { + return err + } + if captionHTML != "" { + if err := w.WriteField("caption", captionHTML); err != nil { + return err + } + if err := w.WriteField("parse_mode", "HTML"); err != nil { + return err + } + } + + part, err := w.CreateFormFile("photo", filename) + if err != nil { + return err + } + if _, err := part.Write(png); err != nil { + return err + } + if err := w.Close(); err != nil { + return err + } + + endpoint := "https://api.telegram.org/bot" + token + "/sendPhoto" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, &buf) + if err != nil { + return err + } + req.Header.Set("Content-Type", w.FormDataContentType()) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + msg := strings.TrimSpace(string(body)) + if len(msg) > 200 { + msg = msg[:200] + } + return fmt.Errorf("sendPhoto HTTP %d: %s", resp.StatusCode, msg) + } + return nil +} + +// runHistorySampler periodically appends a network snapshot for every +// cluster that has an agent installed so long-running non-TUI processes +// (like the bot daemon) can build up data for P95/graph queries. +func runHistorySampler(ctx context.Context, svc *cluster.Service, interval time.Duration, logErr io.Writer) { + if svc == nil || svc.NetworkStore() == nil { + return + } + if interval <= 0 { + interval = 30 * time.Second + } + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + sampleOnce(ctx, svc, logErr) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + sampleOnce(ctx, svc, logErr) + } + } +} + +func sampleOnce(ctx context.Context, svc *cluster.Service, logErr io.Writer) { + clusters, _, err := svc.List() + if err != nil { + return + } + store := svc.NetworkStore() + if store == nil { + return + } + availStore := history.NewAvailabilityStore(svc.DataDir()) + capStore := history.NewCapacityStore(svc.DataDir()) + for _, c := range clusters { + if !c.Agent.Installed { + continue + } + select { + case <-ctx.Done(): + return + default: + } + callCtx, cancel := context.WithTimeout(ctx, 6*time.Second) + stats, err := svc.AgentStatsTyped(callCtx, c.ID) + cancel() + if err != nil { + fmt.Fprintf(logErr, "history sampler: %s: %v\n", c.Name, err) + _ = availStore.Append(c.ID, history.AvailabilitySnapshot{ + Timestamp: time.Now().UTC(), + ClusterUp: false, + Error: err.Error(), + }) + continue + } + items := make([]history.InterfaceSample, 0, len(stats.Network)) + // With a single stats point we don't have rate info; estimate Mbps + // per-interface as a simple counter delta versus the sampler's prior + // snapshot if available. + samplerMu.Lock() + prev := samplerPrev[c.ID] + samplerMu.Unlock() + now := time.Now().UTC() + for _, n := range stats.Network { + rxMbps := 0.0 + txMbps := 0.0 + if prev != nil { + if pn, ok := prev.Counters[n.Interface]; ok { + elapsed := now.Sub(prev.Timestamp).Seconds() + if elapsed > 0.1 { + if n.RxBytes >= pn.RxBytes { + rxMbps = float64(n.RxBytes-pn.RxBytes) * 8 / elapsed / 1_000_000 + } + if n.TxBytes >= pn.TxBytes { + txMbps = float64(n.TxBytes-pn.TxBytes) * 8 / elapsed / 1_000_000 + } + } + } + } + items = append(items, history.InterfaceSample{ + Interface: n.Interface, + RxMbps: rxMbps, + TxMbps: txMbps, + RxDrops: n.RxDrops, + TxDrops: n.TxDrops, + }) + } + counters := make(map[string]samplerIfaceCounter, len(stats.Network)) + for _, n := range stats.Network { + counters[n.Interface] = samplerIfaceCounter{RxBytes: n.RxBytes, TxBytes: n.TxBytes} + } + samplerMu.Lock() + samplerPrev[c.ID] = &samplerState{Timestamp: now, Counters: counters} + samplerMu.Unlock() + + // Only persist when we have a real delta — skip the first bootstrap + // sample per cluster to avoid a meaningless zero row. + if prev != nil { + snap := history.NetworkSnapshot{ + Timestamp: now, + Interfaces: items, + } + if err := store.Append(c.ID, snap); err != nil { + fmt.Fprintf(logErr, "history sampler: append %s: %v\n", c.Name, err) + } + } + // Availability and VM state snapshot. + vmStates := map[string]string{} + vmCtx, vmCancel := context.WithTimeout(ctx, 6*time.Second) + if rawStates, vmErr := svc.ListVMStates(vmCtx, c.ID); vmErr == nil { + vmStates = rawStates + } + vmCancel() + _ = availStore.Append(c.ID, history.AvailabilitySnapshot{ + Timestamp: time.Now().UTC(), + ClusterUp: true, + VMStates: vmStates, + }) + // Capacity snapshot (disk usage trend source). + disks := make([]history.CapacityDiskPoint, 0, len(stats.Disk)) + for _, d := range stats.Disk { + if d.TotalBytes == 0 { + continue + } + disks = append(disks, history.CapacityDiskPoint{ + Mount: d.MountPoint, + UsedBytes: d.UsedBytes, + TotalBytes: d.TotalBytes, + }) + } + if len(disks) > 0 { + _ = capStore.Append(c.ID, history.CapacitySnapshot{ + Timestamp: time.Now().UTC(), + Disks: disks, + }) + } + } +} diff --git a/internal/cli/command_exec.go b/internal/cli/command_exec.go new file mode 100644 index 0000000..1dccbea --- /dev/null +++ b/internal/cli/command_exec.go @@ -0,0 +1,182 @@ +package cli + +import ( + "context" + "errors" + "os/exec" + "regexp" + "strings" + "time" + + "pxmon/internal/cluster" +) + +type observerCommandOptions struct { + AllowShellEscape bool + StatsAutoOnce bool + BlockBotRun bool + StripANSI bool + EmbeddedConsole bool +} + +var ansiEscapeRE = regexp.MustCompile(`\x1b\[[0-9;?]*[ -/]*[@-~]`) + +func stripANSI(s string) string { + if s == "" { + return s + } + return ansiEscapeRE.ReplaceAllString(s, "") +} + +func runObserverScopedCommand(svc *cluster.Service, configPath, line string, opts observerCommandOptions) (string, int) { + out, code := runObserverScopedCommandInner(svc, configPath, line, opts) + if opts.StripANSI { + out = stripANSI(out) + } + return out, code +} + +func runObserverScopedCommandInner(svc *cluster.Service, configPath, line string, opts observerCommandOptions) (string, int) { + line = strings.TrimSpace(line) + if line == "" { + return "empty command", 2 + } + + if strings.HasPrefix(line, "!") { + if !opts.AllowShellEscape { + return "shell escape is disabled for this channel", 2 + } + cmdline := strings.TrimSpace(strings.TrimPrefix(line, "!")) + if cmdline == "" { + return "usage: !", 2 + } + cmd := exec.Command("/bin/sh", "-lc", cmdline) + out, err := cmd.CombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return string(out), exitErr.ExitCode() + } + if len(out) == 0 { + return err.Error(), 1 + } + return string(out), 1 + } + return string(out), 0 + } + + args, err := parseShellArgs(line) + if err != nil || len(args) == 0 { + if err != nil { + return err.Error(), 2 + } + return "empty command", 2 + } + + norm := normalizeObserverConsoleArgs(args) + if len(norm) == 0 { + return "empty command", 2 + } + + if opts.BlockBotRun && len(norm) >= 3 && + strings.EqualFold(norm[0], "bot") && + strings.EqualFold(norm[1], "telegram") && + strings.EqualFold(norm[2], "run") { + return "running `bot telegram run` from bot channel is blocked", 2 + } + + if opts.StatsAutoOnce && len(norm) >= 2 && + strings.EqualFold(norm[0], "cluster") && + strings.EqualFold(norm[1], "stats") && + !hasArg(norm[2:], "--once") { + norm = append(norm, "--once") + } + + if handled, out, code := runPluginArgs(svc, norm); handled { + return out, code + } + + return runObserverCommand(configPath, norm, opts.EmbeddedConsole) +} + +func runPluginArgs(svc *cluster.Service, args []string) (bool, string, int) { + if len(args) == 0 || svc == nil { + return false, "", 0 + } + + tool := strings.ToLower(strings.TrimSpace(args[0])) + switch tool { + case "kvm", "lxc", "lxd", "bird", "frr": + default: + return false, "", 0 + } + + selector, rest, err := parseClusterSelectorArg(args[1:]) + if err != nil { + return true, err.Error(), 2 + } + + action := "" + params := []string{} + if len(rest) > 0 { + action = strings.ToLower(strings.TrimSpace(rest[0])) + params = rest[1:] + } + if action == "" { + switch tool { + case "kvm", "lxc", "lxd": + action = "list" + default: + action = "status" + } + } + if tool == "kvm" && action == "top" { + for _, p := range params { + if strings.EqualFold(strings.TrimSpace(p), "--live") || strings.EqualFold(strings.TrimSpace(p), "-L") { + return true, "kvm top --live is removed; use `kvm top` for allocated VM specs", 2 + } + } + } + + ctx, cancel := context.WithTimeout(context.Background(), pluginActionTimeout(tool, action, false)) + defer cancel() + + out, execErr := svc.RunPluginAction(ctx, selector, tool, action, params) + if execErr != nil { + return true, execErr.Error(), 1 + } + if strings.TrimSpace(out) == "" { + out = "ok" + } + return true, out, 0 +} + +func pluginActionTimeout(tool, action string, live bool) time.Duration { + tool = strings.ToLower(strings.TrimSpace(tool)) + action = strings.ToLower(strings.TrimSpace(action)) + switch tool { + case "kvm": + switch action { + case "top": + if live { + return 2 * time.Minute + } + return 90 * time.Second + case "net-top", "net": + if live { + return 90 * time.Second + } + return 60 * time.Second + default: + return 45 * time.Second + } + case "lxc", "lxd": + if action == "top" || action == "net-top" || action == "net" { + return 45 * time.Second + } + } + if live { + return 35 * time.Second + } + return 25 * time.Second +} diff --git a/internal/cli/extras.go b/internal/cli/extras.go new file mode 100644 index 0000000..c16afb5 --- /dev/null +++ b/internal/cli/extras.go @@ -0,0 +1,2068 @@ +package cli + +import ( + "context" + "encoding/csv" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +func (a *App) runExplain(args []string) int { + fs := flag.NewFlagSet("explain", flag.ContinueOnError) + fs.SetOutput(a.err) + find := fs.String("find", "", "Fast filter text") + if err := fs.Parse(args); err != nil { + return 2 + } + query := strings.ToLower(strings.TrimSpace(*find)) + if query == "" && fs.NArg() > 0 { + query = strings.ToLower(strings.TrimSpace(strings.Join(fs.Args(), " "))) + } + catalog := []string{ + "cluster list", + "cluster show [name]", + "cluster usage [name] --range 1h|1d|1mo|all", + "cluster traffic [name] --range 1h|1d|1mo|all", + "cluster graph [name] --range 1d|1mo|all", + "cluster p95 [name] --iface eth0 --range 30d --graph", + "cluster tag add|rm|ls [name] --tags prod,billing", + "cluster kvm-tag add|rm|ls [name] --vm vm123 --tags critical", + "cluster alert show|set [name]", + "cluster alert-vm show|set|check [name]", + "cluster drift [name]", + "cluster report export --format json|csv --out ./report.json", + "cluster backup target add|ls|rm", + "cluster backup plan add|ls|rm", + "cluster backup run ", + "cluster runbook list|show|run ", + "cluster schedule add|ls|rm|run-due", + "cluster change-history --tail 100", + "cluster agent status", + "cluster agent update [name] --restart-bot=true", + } + if query == "" { + for _, line := range catalog { + fmt.Fprintln(a.out, line) + } + return 0 + } + matched := 0 + for _, line := range catalog { + if strings.Contains(strings.ToLower(line), query) { + fmt.Fprintln(a.out, line) + matched++ + } + } + if matched == 0 { + fmt.Fprintf(a.out, "No explain matches for %q\n", query) + return 1 + } + return 0 +} + +func parseTagsCSV(v string) []string { + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + out = append(out, p) + } + return out +} + +func parseSinceRange(raw string) (time.Time, string, error) { + r := strings.ToLower(strings.TrimSpace(raw)) + now := time.Now().UTC() + switch r { + case "", "30d", "1mo", "month": + return now.Add(-30 * 24 * time.Hour), "30d", nil + case "1d", "24h", "day": + return now.Add(-24 * time.Hour), "1d", nil + case "7d", "week": + return now.Add(-7 * 24 * time.Hour), "7d", nil + case "all": + return time.Time{}, "all", nil + default: + d, err := time.ParseDuration(r) + if err != nil { + return time.Time{}, "", fmt.Errorf("invalid range %q (use 1d|7d|30d|all)", raw) + } + return now.Add(-d), r, nil + } +} + +type multiStringFlag []string + +func (m *multiStringFlag) String() string { + return strings.Join(*m, ",") +} + +func (m *multiStringFlag) Set(v string) error { + *m = append(*m, strings.TrimSpace(v)) + return nil +} + +func (a *App) runClusterTag(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster tag [name-or-id]") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + switch sub { + case "ls", "list", "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + if strings.TrimSpace(selector) != "" { + c, err := svc.Get(selector) + if err != nil { + fmt.Fprintf(a.err, "cluster tag ls: %v\n", err) + return 1 + } + tags := c.Tags + if jsonOut { + _ = writeJSON(a.out, map[string]any{"cluster": c.Name, "tags": tags}) + return 0 + } + fmt.Fprintf(a.out, "Cluster: %s\n", c.Name) + if len(tags) == 0 { + fmt.Fprintln(a.out, "Tags: (none)") + } else { + fmt.Fprintf(a.out, "Tags: %s\n", strings.Join(tags, ", ")) + } + return 0 + } + clusters, _, err := svc.List() + if err != nil { + fmt.Fprintf(a.err, "cluster tag ls: %v\n", err) + return 1 + } + type row struct { + Cluster string `json:"cluster"` + Tags []string `json:"tags"` + } + rows := make([]row, 0, len(clusters)) + for _, c := range clusters { + rows = append(rows, row{Cluster: c.Name, Tags: c.Tags}) + } + if jsonOut { + _ = writeJSON(a.out, rows) + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "CLUSTER\tTAGS") + for _, r := range rows { + v := "(none)" + if len(r.Tags) > 0 { + v = strings.Join(r.Tags, ", ") + } + fmt.Fprintf(tw, "%s\t%s\n", r.Cluster, v) + } + _ = tw.Flush() + return 0 + case "add", "rm", "remove", "del": + fs := flag.NewFlagSet("cluster tag "+sub, flag.ContinueOnError) + fs.SetOutput(a.err) + tagsCSV := fs.String("tags", "", "Comma separated tags") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster tag add|rm [name-or-id] --tags a,b") + return 2 + } + selector = fs.Arg(0) + } + tags := parseTagsCSV(*tagsCSV) + if len(tags) == 0 { + tags = fs.Args() + } + if len(tags) == 0 { + fmt.Fprintln(a.err, "cluster tag: provide tags with --tags a,b") + return 2 + } + var ( + c cluster.Cluster + err error + ) + if sub == "add" { + c, err = svc.AddClusterTags(selector, tags) + } else { + c, err = svc.RemoveClusterTags(selector, tags) + } + if err != nil { + fmt.Fprintf(a.err, "cluster tag %s: %v\n", sub, err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"cluster": c.Name, "tags": c.Tags}) + return 0 + } + fmt.Fprintf(a.out, "%s tags for %s: %s\n", strings.ToUpper(sub), c.Name, strings.Join(c.Tags, ", ")) + return 0 + default: + fmt.Fprintf(a.err, "unknown tag subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterKVMTag(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster kvm-tag [name-or-id] --vm [--tags a,b]") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + fs := flag.NewFlagSet("cluster kvm-tag "+sub, flag.ContinueOnError) + fs.SetOutput(a.err) + vm := fs.String("vm", "", "KVM domain name") + tagsCSV := fs.String("tags", "", "Comma separated tags") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster kvm-tag [name-or-id] --vm [--tags a,b]") + return 2 + } + selector = fs.Arg(0) + } + vmName := strings.TrimSpace(*vm) + switch sub { + case "ls", "list", "show": + items, err := svc.ListKVMTags(selector, vmName) + if err != nil { + fmt.Fprintf(a.err, "kvm-tag ls: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no vm tags)") + return 0 + } + keys := make([]string, 0, len(items)) + for k := range items { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(a.out, "%s: %s\n", k, strings.Join(items[k], ", ")) + } + return 0 + case "add", "rm", "remove", "del": + if vmName == "" { + fmt.Fprintln(a.err, "kvm-tag: --vm is required") + return 2 + } + tags := parseTagsCSV(*tagsCSV) + if len(tags) == 0 { + tags = fs.Args() + } + if len(tags) == 0 { + fmt.Fprintln(a.err, "kvm-tag: provide tags with --tags a,b") + return 2 + } + var ( + c cluster.Cluster + err error + ) + if sub == "add" { + c, err = svc.AddKVMTag(selector, vmName, tags) + } else { + c, err = svc.RemoveKVMTag(selector, vmName, tags) + } + if err != nil { + fmt.Fprintf(a.err, "kvm-tag %s: %v\n", sub, err) + return 1 + } + vmTags := c.KVMTags[vmName] + if jsonOut { + _ = writeJSON(a.out, map[string]any{"cluster": c.Name, "vm": vmName, "tags": vmTags}) + return 0 + } + fmt.Fprintf(a.out, "%s vm tags for %s/%s: %s\n", strings.ToUpper(sub), c.Name, vmName, strings.Join(vmTags, ", ")) + return 0 + default: + fmt.Fprintf(a.err, "unknown kvm-tag subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterVMAlert(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster alert-vm [name-or-id]") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + switch sub { + case "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + p, err := svc.GetVMAlertPolicy(selector) + if err != nil { + fmt.Fprintf(a.err, "alert-vm show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, p) + return 0 + } + fmt.Fprintf(a.out, "enabled=%t warn_on_shutoff=%t min_running=%d\n", p.Enabled, p.WarnOnShutoff, p.MinRunning) + return 0 + case "set": + fs := flag.NewFlagSet("cluster alert-vm set", flag.ContinueOnError) + fs.SetOutput(a.err) + enabled := fs.Bool("enabled", false, "Enable VM state alerts") + warnOnShutoff := fs.Bool("warn-on-shutoff", true, "Warn when any VM is shut off") + minRunning := fs.Int("min-running", 1, "Minimum expected running VM count") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector != "" { + fmt.Fprintln(a.err, "usage: pxmon cluster alert-vm set [name-or-id] [--enabled --warn-on-shutoff --min-running 1]") + return 2 + } + selector = fs.Arg(0) + } + current, err := svc.GetVMAlertPolicy(selector) + if err != nil { + fmt.Fprintf(a.err, "alert-vm set: %v\n", err) + return 1 + } + changed := false + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "enabled": + current.Enabled = *enabled + changed = true + case "warn-on-shutoff": + current.WarnOnShutoff = *warnOnShutoff + changed = true + case "min-running": + current.MinRunning = *minRunning + changed = true + } + }) + if !changed { + fmt.Fprintln(a.err, "alert-vm set: provide at least one flag to change") + return 2 + } + c, err := svc.SetVMAlertPolicy(selector, current) + if err != nil { + fmt.Fprintf(a.err, "alert-vm set: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, c.VMAlerts) + return 0 + } + fmt.Fprintf(a.out, "Updated vm alerts for %s: enabled=%t warn_on_shutoff=%t min_running=%d\n", c.Name, c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning) + return 0 + case "check": + selector := "" + if len(args) > 1 { + selector = args[1] + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + rep, err := svc.CheckVMAlerts(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "alert-vm check: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, rep) + if len(rep.Warnings) > 0 { + return 1 + } + return 0 + } + fmt.Fprintf(a.out, "VMs total=%d running=%d shut_off=%d paused=%d others=%d\n", rep.Total, rep.Running, rep.ShutOff, rep.Paused, rep.Others) + if len(rep.ShutOffNames) > 0 { + fmt.Fprintf(a.out, "shut off: %s\n", strings.Join(rep.ShutOffNames, ", ")) + } + if len(rep.PausedNames) > 0 { + fmt.Fprintf(a.out, "paused: %s\n", strings.Join(rep.PausedNames, ", ")) + } + if len(rep.OtherNames) > 0 { + fmt.Fprintf(a.out, "other: %s\n", strings.Join(rep.OtherNames, ", ")) + } + if len(rep.Warnings) > 0 { + for _, w := range rep.Warnings { + fmt.Fprintf(a.out, "! %s\n", w) + } + return 1 + } + fmt.Fprintln(a.out, "VM alerts OK") + return 0 + default: + fmt.Fprintf(a.err, "unknown alert-vm subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterAlertRouting(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster alert-routing [cluster]") + return 2 + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + p, err := svc.GetAlertRouting(selector) + if err != nil { + fmt.Fprintf(a.err, "alert-routing show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, p) + return 0 + } + fmt.Fprintf(a.out, "critical_immediate=%t warning_batch_mins=%d\n", p.CriticalImmediate, p.WarningBatchMins) + return 0 + case "set": + fs := flag.NewFlagSet("cluster alert-routing set", flag.ContinueOnError) + fs.SetOutput(a.err) + critical := fs.Bool("critical-immediate", true, "Send critical alerts immediately") + batch := fs.Int("warning-batch-mins", 5, "Warning batching interval in minutes") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 && selector == "" { + selector = fs.Arg(0) + } + cur, err := svc.GetAlertRouting(selector) + if err != nil { + fmt.Fprintf(a.err, "alert-routing set: %v\n", err) + return 1 + } + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "critical-immediate": + cur.CriticalImmediate = *critical + case "warning-batch-mins": + cur.WarningBatchMins = *batch + } + }) + c, err := svc.SetAlertRouting(selector, cur) + if err != nil { + fmt.Fprintf(a.err, "alert-routing set: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, c.AlertRouting) + return 0 + } + fmt.Fprintf(a.out, "Updated alert routing for %s: critical_immediate=%t warning_batch_mins=%d\n", + c.Name, c.AlertRouting.CriticalImmediate, c.AlertRouting.WarningBatchMins) + return 0 + default: + fmt.Fprintf(a.err, "unknown alert-routing subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterRunbookTrigger(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster runbook-trigger [cluster]") + return 2 + } + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "show": + selector := "" + if len(args) > 1 { + selector = args[1] + } + p, err := svc.GetRunbookTrigger(selector) + if err != nil { + fmt.Fprintf(a.err, "runbook-trigger show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, p) + return 0 + } + fmt.Fprintf(a.out, "enabled=%t on_vm_shutoff=%t runbook_id=%s cooldown_mins=%d last_triggered=%s\n", + p.Enabled, p.OnVMShutoff, emptyFallback(p.RunbookID, "-"), p.CooldownMins, + emptyFallback(p.LastTriggered.Format(time.RFC3339), "-")) + return 0 + case "set": + fs := flag.NewFlagSet("cluster runbook-trigger set", flag.ContinueOnError) + fs.SetOutput(a.err) + enabled := fs.Bool("enabled", false, "Enable auto-trigger") + onShutoff := fs.Bool("on-vm-shutoff", true, "Trigger on VM shutoff alert") + runbookID := fs.String("runbook-id", "", "Runbook ID to execute") + cooldown := fs.Int("cooldown-mins", 30, "Cooldown between triggers") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 && selector == "" { + selector = fs.Arg(0) + } + cur, err := svc.GetRunbookTrigger(selector) + if err != nil { + fmt.Fprintf(a.err, "runbook-trigger set: %v\n", err) + return 1 + } + fs.Visit(func(f *flag.Flag) { + switch f.Name { + case "enabled": + cur.Enabled = *enabled + case "on-vm-shutoff": + cur.OnVMShutoff = *onShutoff + case "runbook-id": + cur.RunbookID = strings.TrimSpace(*runbookID) + case "cooldown-mins": + cur.CooldownMins = *cooldown + } + }) + c, err := svc.SetRunbookTrigger(selector, cur) + if err != nil { + fmt.Fprintf(a.err, "runbook-trigger set: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, c.RunbookTrigger) + return 0 + } + fmt.Fprintf(a.out, "Updated runbook trigger for %s: enabled=%t on_vm_shutoff=%t runbook_id=%s cooldown_mins=%d\n", + c.Name, c.RunbookTrigger.Enabled, c.RunbookTrigger.OnVMShutoff, + emptyFallback(c.RunbookTrigger.RunbookID, "-"), c.RunbookTrigger.CooldownMins) + return 0 + default: + fmt.Fprintf(a.err, "unknown runbook-trigger subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterP95(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster p95", flag.ContinueOnError) + fs.SetOutput(a.err) + iface := fs.String("iface", "", "Interface name (required)") + rangeStr := fs.String("range", "30d", "Time window: 1h|1d|30d|1mo|all") + graph := fs.Bool("graph", false, "Generate graph PNG") + outPath := fs.String("out", "", "Output PNG path for --graph") + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 { + if selector == "" { + selector = fs.Arg(0) + } else if strings.TrimSpace(*iface) == "" { + *iface = fs.Arg(0) + } + } + if fs.NArg() > 1 && strings.TrimSpace(*iface) == "" { + *iface = fs.Arg(1) + } + if strings.TrimSpace(*iface) == "" { + fmt.Fprintln(a.err, "cluster p95: --iface is required") + return 2 + } + rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr)) + if !ok { + fmt.Fprintf(a.err, "cluster p95: invalid --range %q\n", *rangeStr) + return 2 + } + snap, err := svc.CollectInterfaceP95(selector, *iface, rng) + if err != nil { + fmt.Fprintf(a.err, "cluster p95: %v\n", err) + return 1 + } + pngPath := "" + if *graph { + png, err := svc.RenderInterfaceP95GraphPNG(snap) + if err != nil { + fmt.Fprintf(a.err, "cluster p95: render graph: %v\n", err) + return 1 + } + pngPath = strings.TrimSpace(*outPath) + if pngPath == "" { + pngPath = fmt.Sprintf("%s-%s-%s.png", sanitizeFilename(snap.ClusterName), sanitizeFilename(snap.Interface), rng) + } + if err := os.WriteFile(pngPath, png, 0o600); err != nil { + fmt.Fprintf(a.err, "cluster p95: write graph: %v\n", err) + return 1 + } + } + if jsonOut { + resp := map[string]any{ + "cluster": snap.ClusterName, + "interface": snap.Interface, + "range": rng, + "samples": snap.Samples, + "p95_mbps": snap.P95Mbps, + "avg_mbps": snap.AvgMbps, + "max_mbps": snap.MaxMbps, + } + if pngPath != "" { + resp["graph"] = pngPath + } + _ = writeJSON(a.out, resp) + return 0 + } + if pngPath != "" { + fmt.Fprintf(a.out, "Saved: %s\n", pngPath) + } + reportPath := fmt.Sprintf("%s-%s-%s.txt", + sanitizeFilename(snap.ClusterName), + sanitizeFilename(snap.Interface), + time.Now().UTC().Format("20060102")) + reportBody := strings.Join([]string{ + fmt.Sprintf("cluster=%s", snap.ClusterName), + fmt.Sprintf("iface=%s", snap.Interface), + fmt.Sprintf("range=%s", rng), + fmt.Sprintf("generated_at=%s", time.Now().UTC().Format(time.RFC3339)), + fmt.Sprintf("p95_mbps=%.3f", snap.P95Mbps), + fmt.Sprintf("avg_mbps=%.3f", snap.AvgMbps), + fmt.Sprintf("max_mbps=%.3f", snap.MaxMbps), + fmt.Sprintf("samples=%d", snap.Samples), + "", + }, "\n") + if err := os.WriteFile(reportPath, []byte(reportBody), 0o600); err == nil { + fmt.Fprintf(a.out, "Saved: %s\n", reportPath) + } + fmt.Fprintf(a.out, "Cluster: %s\n", snap.ClusterName) + fmt.Fprintf(a.out, "Iface: %s\n", snap.Interface) + fmt.Fprintf(a.out, "Range: %s\n", rng.Label()) + fmt.Fprintf(a.out, "P95: %s over %d samples\n", formatMbpsHuman(snap.P95Mbps), snap.Samples) + fmt.Fprintf(a.out, "Max: %s\n", formatMbpsHuman(snap.MaxMbps)) + fmt.Fprintf(a.out, "Avg: %s\n", formatMbpsHuman(snap.AvgMbps)) + _ = svc.AppendChange("p95", snap.ClusterName, fmt.Sprintf("iface=%s range=%s p95=%.2f", snap.Interface, rng, snap.P95Mbps)) + return 0 +} + +func (a *App) runClusterSLO(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster slo", flag.ContinueOnError) + fs.SetOutput(a.err) + rangeStr := fs.String("range", "30d", "Time window: 1d|7d|30d|all") + vm := fs.String("vm", "", "Optional VM name filter") + selector, parseArgs := splitLeadingSelector(args) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 && selector == "" { + selector = fs.Arg(0) + } + since, label, err := parseSinceRange(*rangeStr) + if err != nil { + fmt.Fprintf(a.err, "cluster slo: %v\n", err) + return 2 + } + rep, err := svc.AvailabilityReport(selector, since, *vm) + if err != nil { + fmt.Fprintf(a.err, "cluster slo: %v\n", err) + return 1 + } + rep.Range = label + if jsonOut { + _ = writeJSON(a.out, rep) + return 0 + } + fmt.Fprintf(a.out, "Cluster: %s\n", rep.Cluster) + fmt.Fprintf(a.out, "Range: %s\n", rep.Range) + fmt.Fprintf(a.out, "Uptime: %.2f%% (%d/%d samples up)\n", rep.Availability, rep.UpSamples, rep.Samples) + if strings.TrimSpace(*vm) != "" { + if len(rep.VMs) == 0 { + fmt.Fprintf(a.out, "VM %s: no samples\n", *vm) + } else { + v := rep.VMs[0] + fmt.Fprintf(a.out, "VM %s uptime: %.2f%% (%d/%d running)\n", v.Name, v.Availability, v.Running, v.Samples) + } + } else if len(rep.VMs) > 0 { + maxRows := 10 + if len(rep.VMs) < maxRows { + maxRows = len(rep.VMs) + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "VM\tUPTIME\tRUNNING/SAMPLES") + for i := 0; i < maxRows; i++ { + v := rep.VMs[i] + fmt.Fprintf(tw, "%s\t%.2f%%\t%d/%d\n", v.Name, v.Availability, v.Running, v.Samples) + } + _ = tw.Flush() + if len(rep.VMs) > maxRows { + fmt.Fprintf(a.out, "... +%d VM(s)\n", len(rep.VMs)-maxRows) + } + } + return 0 +} + +func (a *App) runClusterCapacity(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 || strings.EqualFold(strings.TrimSpace(args[0]), "forecast") { + rest := args + if len(rest) > 0 && strings.EqualFold(strings.TrimSpace(rest[0]), "forecast") { + rest = rest[1:] + } + fs := flag.NewFlagSet("cluster capacity forecast", flag.ContinueOnError) + fs.SetOutput(a.err) + rangeStr := fs.String("range", "30d", "Time window: 7d|30d|all") + selector, parseArgs := splitLeadingSelector(rest) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 && selector == "" { + selector = fs.Arg(0) + } + since, label, err := parseSinceRange(*rangeStr) + if err != nil { + fmt.Fprintf(a.err, "cluster capacity: %v\n", err) + return 2 + } + rep, err := svc.CapacityForecast(selector, since) + if err != nil { + fmt.Fprintf(a.err, "cluster capacity: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"range": label, "report": rep}) + return 0 + } + fmt.Fprintf(a.out, "Cluster: %s\n", rep.Cluster) + fmt.Fprintf(a.out, "Range: %s\n", label) + fmt.Fprintf(a.out, "Samples: %d\n", rep.Samples) + if len(rep.Items) == 0 { + fmt.Fprintln(a.out, "No capacity trend data yet") + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "MOUNT\tUSED\tDAYS_TO_90%\tDAYS_TO_95%\tSLOPE") + for _, it := range rep.Items { + d90 := "inf" + d95 := "inf" + if !math.IsInf(it.DaysTo90, 1) { + d90 = fmt.Sprintf("%.1f", it.DaysTo90) + } + if !math.IsInf(it.DaysTo95, 1) { + d95 = fmt.Sprintf("%.1f", it.DaysTo95) + } + fmt.Fprintf(tw, "%s\t%.1f%%\t%s\t%s\t%.2f MiB/day\n", + it.Mount, it.UsedPct, d90, d95, it.SlopeBytesSec*86400/1024/1024) + } + _ = tw.Flush() + return 0 + } + fmt.Fprintln(a.err, "usage: pxmon cluster capacity forecast [cluster] [--range 7d|30d|all]") + return 2 +} + +func (a *App) runClusterDrift(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) > 0 { + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "baseline": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster drift baseline [cluster]") + return 2 + } + switch strings.ToLower(strings.TrimSpace(args[1])) { + case "set": + selector := "" + if len(args) > 2 { + selector = args[2] + } + c, err := svc.SetDriftBaseline(selector) + if err != nil { + fmt.Fprintf(a.err, "drift baseline set: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, c.Drift) + return 0 + } + fmt.Fprintf(a.out, "Baseline set for %s\n", c.Name) + return 0 + case "show": + selector := "" + if len(args) > 2 { + selector = args[2] + } + d, err := svc.GetDriftControl(selector) + if err != nil { + fmt.Fprintf(a.err, "drift baseline show: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, d.Baseline) + return 0 + } + fmt.Fprintf(a.out, "enabled=%t set_at=%s agent=%s software=%s\n", + d.Baseline.Enabled, + emptyFallback(d.Baseline.SetAt.Format(time.RFC3339), "-"), + emptyFallback(d.Baseline.AgentVersion, "-"), + emptyFallback(d.Baseline.Software, "-")) + return 0 + } + fmt.Fprintf(a.err, "unknown drift baseline subcommand %q\n", args[1]) + return 2 + case "ack": + fs := flag.NewFlagSet("cluster drift ack", flag.ContinueOnError) + fs.SetOutput(a.err) + kind := fs.String("kind", "", "Issue kind to ack (e.g. agent_version)") + forDur := fs.Duration("for", 24*time.Hour, "Ack duration (e.g. 24h)") + selector, parseArgs := splitLeadingSelector(args[1:]) + if err := fs.Parse(parseArgs); err != nil { + return 2 + } + if fs.NArg() > 0 && selector == "" { + selector = fs.Arg(0) + } + if strings.TrimSpace(*kind) == "" { + fmt.Fprintln(a.err, "drift ack: --kind is required") + return 2 + } + until := time.Now().UTC().Add(*forDur) + _, err := svc.AckDriftIssue(selector, *kind, until) + if err != nil { + fmt.Fprintf(a.err, "drift ack: %v\n", err) + return 1 + } + fmt.Fprintf(a.out, "Acked %s until %s\n", *kind, until.Format(time.RFC3339)) + return 0 + } + } + + selector := "" + if len(args) > 0 { + selector = args[0] + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + rep, err := svc.DetectDrift(ctx, selector) + if err != nil { + fmt.Fprintf(a.err, "cluster drift: %v\n", err) + return 1 + } + c, _ := svc.Get(selector) + filtered := make([]cluster.DriftIssue, 0, len(rep.Issues)) + for _, it := range rep.Issues { + if svc.IsDriftIssueAcked(c, it.Kind, time.Now().UTC()) { + continue + } + filtered = append(filtered, it) + } + rep.Issues = filtered + if jsonOut { + _ = writeJSON(a.out, rep) + if len(rep.Issues) > 0 { + return 1 + } + return 0 + } + fmt.Fprintf(a.out, "Drift report for %s (%s)\n", rep.Cluster, rep.Generated.Format(time.RFC3339)) + if len(rep.Issues) == 0 { + fmt.Fprintln(a.out, "No drift detected") + return 0 + } + for _, it := range rep.Issues { + fmt.Fprintf(a.out, "[%s] %s: %s\n", strings.ToUpper(it.Level), it.Kind, it.Message) + } + return 1 +} + +func (a *App) runClusterChangeHistory(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster change-history", flag.ContinueOnError) + fs.SetOutput(a.err) + tail := fs.Int("tail", 100, "Last N entries") + if err := fs.Parse(args); err != nil { + return 2 + } + items, err := svc.ListChanges(*tail) + if err != nil { + fmt.Fprintf(a.err, "cluster change-history: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no history)") + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "TIME\tACTION\tTARGET\tDETAILS") + for _, it := range items { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", it.At.Format("2006-01-02 15:04:05"), it.Action, it.Target, it.Details) + } + _ = tw.Flush() + return 0 +} + +func (a *App) runClusterReport(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster report export --out ./report.json [--format json|csv]") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + if sub != "export" { + fmt.Fprintf(a.err, "unknown report subcommand %q\n", args[0]) + return 2 + } + fs := flag.NewFlagSet("cluster report export", flag.ContinueOnError) + fs.SetOutput(a.err) + format := fs.String("format", "json", "Output format: json|csv") + out := fs.String("out", "", "Output file path") + probe := fs.Bool("probe", false, "Probe agent reachability for each cluster") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + if strings.TrimSpace(*out) == "" { + fmt.Fprintln(a.err, "cluster report export: --out is required") + return 2 + } + clusters, activeID, err := svc.List() + if err != nil { + fmt.Fprintf(a.err, "cluster report export: %v\n", err) + return 1 + } + type row struct { + Name string `json:"name"` + ID string `json:"id"` + Host string `json:"host"` + User string `json:"user"` + Active bool `json:"active"` + Agent string `json:"agent"` + AgentVersion string `json:"agent_version"` + Software string `json:"software"` + Tags []string `json:"tags,omitempty"` + UpdatedAt string `json:"updated_at"` + Reachable string `json:"reachable,omitempty"` + } + items := make([]row, 0, len(clusters)) + for _, c := range clusters { + r := row{ + Name: c.Name, + ID: c.ID, + Host: c.Host, + User: c.User, + Active: c.ID == activeID, + Agent: ternary(c.Agent.Installed, "installed", "none"), + AgentVersion: c.Agent.Version, + Software: c.Software.Summary(), + Tags: c.Tags, + UpdatedAt: c.UpdatedAt.Format(time.RFC3339), + } + if *probe && c.Agent.Installed { + ctx, cancel := context.WithTimeout(context.Background(), 1800*time.Millisecond) + p, err := svc.PingAgent(ctx, c.ID) + cancel() + if err == nil && p.Reachable && p.StatusCode < 400 { + r.Reachable = "up" + } else { + r.Reachable = "down" + } + } + items = append(items, r) + } + if err := os.MkdirAll(filepath.Dir(*out), 0o700); err != nil { + fmt.Fprintf(a.err, "cluster report export: %v\n", err) + return 1 + } + switch strings.ToLower(strings.TrimSpace(*format)) { + case "json": + b, _ := jsonMarshalIndent(map[string]any{"generated_at": time.Now().UTC().Format(time.RFC3339), "clusters": items}) + if err := os.WriteFile(*out, append(b, '\n'), 0o600); err != nil { + fmt.Fprintf(a.err, "cluster report export: %v\n", err) + return 1 + } + case "csv": + f, err := os.Create(*out) + if err != nil { + fmt.Fprintf(a.err, "cluster report export: %v\n", err) + return 1 + } + w := csv.NewWriter(f) + _ = w.Write([]string{"name", "id", "host", "user", "active", "agent", "agent_version", "software", "tags", "updated_at", "reachable"}) + for _, r := range items { + _ = w.Write([]string{r.Name, r.ID, r.Host, r.User, fmt.Sprintf("%t", r.Active), r.Agent, r.AgentVersion, r.Software, strings.Join(r.Tags, ","), r.UpdatedAt, r.Reachable}) + } + w.Flush() + _ = f.Close() + if err := w.Error(); err != nil { + fmt.Fprintf(a.err, "cluster report export: %v\n", err) + return 1 + } + default: + fmt.Fprintf(a.err, "cluster report export: invalid --format %q\n", *format) + return 2 + } + _ = svc.AppendChange("report.export", *out, *format) + if !jsonOut { + fmt.Fprintf(a.out, "Report exported: %s\n", *out) + } + return 0 +} + +func jsonMarshalIndent(v any) ([]byte, error) { + return json.MarshalIndent(v, "", " ") +} + +func (a *App) runClusterBackup(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup ") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + switch sub { + case "target", "targets": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup target ") + return 2 + } + tSub := strings.ToLower(strings.TrimSpace(args[1])) + switch tSub { + case "ls", "list", "show": + items, err := svc.BackupListTargets() + if err != nil { + fmt.Fprintf(a.err, "backup target ls: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no backup targets)") + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tNAME\tTYPE\tENABLED\tDEST") + for _, t := range items { + dest := "-" + if t.Type == "sftp" { + dest = fmt.Sprintf("%s@%s:%s", t.SFTPUser, t.SFTPHost, strings.TrimSpace(t.SFTPBasePath)) + } else if t.Type == "s3" { + dest = fmt.Sprintf("%s/%s", t.S3Endpoint, t.S3Bucket) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%t\t%s\n", t.ID, t.Name, t.Type, t.Enabled, dest) + } + _ = tw.Flush() + return 0 + case "add": + fs := flag.NewFlagSet("cluster backup target add", flag.ContinueOnError) + fs.SetOutput(a.err) + name := fs.String("name", "", "Target name") + tp := fs.String("type", "", "Target type: sftp|s3") + enabled := fs.Bool("enabled", true, "Enable target") + sftpHost := fs.String("sftp-host", "", "SFTP host") + sftpPort := fs.Int("sftp-port", 22, "SFTP port") + sftpUser := fs.String("sftp-user", "", "SFTP user") + sftpPass := fs.String("sftp-password", "", "SFTP password") + sftpKey := fs.String("sftp-key", "", "SFTP key path") + sftpBase := fs.String("sftp-base", "", "SFTP base directory") + s3Endpoint := fs.String("s3-endpoint", "", "S3 endpoint") + s3Region := fs.String("s3-region", "us-east-1", "S3 region") + s3Bucket := fs.String("s3-bucket", "", "S3 bucket") + s3Prefix := fs.String("s3-prefix", "", "S3 key prefix") + s3Access := fs.String("s3-access-key", "", "S3 access key") + s3Secret := fs.String("s3-secret-key", "", "S3 secret key") + s3SSL := fs.Bool("s3-ssl", true, "Use TLS for S3") + s3PathStyle := fs.Bool("s3-path-style", false, "Use S3 path-style URLs") + if err := fs.Parse(args[2:]); err != nil { + return 2 + } + t, err := svc.BackupAddTarget(cluster.BackupTarget{ + Name: strings.TrimSpace(*name), + Type: strings.ToLower(strings.TrimSpace(*tp)), + Enabled: *enabled, + SFTPHost: strings.TrimSpace(*sftpHost), + SFTPPort: *sftpPort, + SFTPUser: strings.TrimSpace(*sftpUser), + SFTPPassword: strings.TrimSpace(*sftpPass), + SFTPKeyPath: strings.TrimSpace(*sftpKey), + SFTPBasePath: strings.TrimSpace(*sftpBase), + S3Endpoint: strings.TrimSpace(*s3Endpoint), + S3Region: strings.TrimSpace(*s3Region), + S3Bucket: strings.TrimSpace(*s3Bucket), + S3Prefix: strings.TrimSpace(*s3Prefix), + S3AccessKey: strings.TrimSpace(*s3Access), + S3SecretKey: strings.TrimSpace(*s3Secret), + S3UseSSL: *s3SSL, + S3PathStyle: *s3PathStyle, + }) + if err != nil { + fmt.Fprintf(a.err, "backup target add: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, t) + return 0 + } + fmt.Fprintf(a.out, "Backup target added: %s (%s)\n", t.Name, t.ID) + return 0 + case "rm", "remove", "del": + if len(args) < 3 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup target rm ") + return 2 + } + t, err := svc.BackupRemoveTarget(args[2]) + if err != nil { + fmt.Fprintf(a.err, "backup target rm: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, t) + return 0 + } + fmt.Fprintf(a.out, "Backup target removed: %s (%s)\n", t.Name, t.ID) + return 0 + case "test": + if len(args) < 3 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup target test ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + msg, err := svc.BackupTestTarget(ctx, args[2]) + if err != nil { + fmt.Fprintf(a.err, "backup target test: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"ok": true, "message": msg}) + return 0 + } + fmt.Fprintf(a.out, "Backup target OK: %s\n", msg) + return 0 + default: + fmt.Fprintf(a.err, "unknown backup target subcommand %q\n", args[1]) + return 2 + } + case "plan", "plans": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup plan ") + return 2 + } + pSub := strings.ToLower(strings.TrimSpace(args[1])) + switch pSub { + case "ls", "list", "show": + items, err := svc.BackupListPlans() + if err != nil { + fmt.Fprintf(a.err, "backup plan ls: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no backup plans)") + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tNAME\tCLUSTER\tTARGET\tPATHS\tEVERY\tENABLED\tLAST_STATUS\tLAST_RUN") + for _, p := range items { + lastRun := "-" + if !p.LastRunAt.IsZero() { + lastRun = p.LastRunAt.Format("2006-01-02 15:04:05") + } + clusterSel := p.Cluster + if strings.TrimSpace(clusterSel) == "" { + clusterSel = "(active)" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%t\t%s\t%s\n", + p.ID, p.Name, clusterSel, p.TargetID, strings.Join(p.Paths, ","), p.Every, p.Enabled, p.LastStatus, lastRun) + } + _ = tw.Flush() + return 0 + case "add": + fs := flag.NewFlagSet("cluster backup plan add", flag.ContinueOnError) + fs.SetOutput(a.err) + name := fs.String("name", "", "Plan name") + clusterSel := fs.String("cluster", "", "Cluster selector (default active)") + target := fs.String("target", "", "Target id or name") + every := fs.String("every", "24h", "Backup interval, e.g. 6h") + retain := fs.Int("retain-days", 30, "Retention days metadata") + enabled := fs.Bool("enabled", true, "Enable plan") + schedule := fs.Bool("schedule", true, "Create scheduler task automatically") + pathList := multiStringFlag{} + fs.Var(&pathList, "path", "Remote path (repeatable)") + if err := fs.Parse(args[2:]); err != nil { + return 2 + } + paths := normalizeCLIPathList(pathList) + p, err := svc.BackupAddPlan(cluster.BackupPlan{ + Name: strings.TrimSpace(*name), + Cluster: strings.TrimSpace(*clusterSel), + TargetID: strings.TrimSpace(*target), + Paths: paths, + Every: strings.TrimSpace(*every), + RetainDays: *retain, + Enabled: *enabled, + }) + if err != nil { + fmt.Fprintf(a.err, "backup plan add: %v\n", err) + return 1 + } + if *schedule { + taskName := "backup-" + p.ID + cmd := "cluster backup run " + p.ID + _, sErr := svc.SchedulerAdd(taskName, "", cmd, p.Every, "observer", "30s", 5, 3, p.Enabled) + if sErr != nil && !strings.Contains(strings.ToLower(sErr.Error()), "already exists") { + fmt.Fprintf(a.err, "backup plan add: schedule create failed: %v\n", sErr) + return 1 + } + if _, running, _, stErr := schedulerDaemonStatus(svc); stErr == nil && !running { + _, _, _ = startSchedulerDaemon(svc, 30*time.Second) + } + } + if jsonOut { + _ = writeJSON(a.out, p) + return 0 + } + fmt.Fprintf(a.out, "Backup plan added: %s (%s)\n", p.Name, p.ID) + if *schedule { + fmt.Fprintf(a.out, "Scheduler task: backup-%s\n", p.ID) + } + return 0 + case "rm", "remove", "del": + if len(args) < 3 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup plan rm ") + return 2 + } + p, err := svc.BackupRemovePlan(args[2]) + if err != nil { + fmt.Fprintf(a.err, "backup plan rm: %v\n", err) + return 1 + } + _, _ = svc.SchedulerRemove("backup-" + p.ID) + if jsonOut { + _ = writeJSON(a.out, p) + return 0 + } + fmt.Fprintf(a.out, "Backup plan removed: %s (%s)\n", p.Name, p.ID) + return 0 + case "run": + if len(args) < 3 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup plan run ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + res, err := svc.BackupRunPlan(ctx, args[2]) + if err != nil { + fmt.Fprintf(a.err, "backup plan run: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, res) + return 0 + } + fmt.Fprintf(a.out, "Backup uploaded: %s\n", res.UploadedTo) + fmt.Fprintf(a.out, "Archive: %s (%d bytes)\n", res.ArchiveName, res.SizeBytes) + return 0 + default: + fmt.Fprintf(a.err, "unknown backup plan subcommand %q\n", args[1]) + return 2 + } + case "run": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster backup run ") + return 2 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + res, err := svc.BackupRunPlan(ctx, args[1]) + if err != nil { + fmt.Fprintf(a.err, "backup run: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, res) + return 0 + } + fmt.Fprintf(a.out, "Backup uploaded: %s\n", res.UploadedTo) + fmt.Fprintf(a.out, "Archive: %s (%d bytes)\n", res.ArchiveName, res.SizeBytes) + return 0 + default: + fmt.Fprintf(a.err, "unknown backup subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterRunbook(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster runbook [id]") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + switch sub { + case "list", "ls": + items, err := svc.ListRunbooks() + if err != nil { + fmt.Fprintf(a.err, "runbook list: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + for _, rb := range items { + fmt.Fprintf(a.out, "%s\t%s\n", rb.ID, rb.Name) + } + return 0 + case "add": + fs := flag.NewFlagSet("cluster runbook add", flag.ContinueOnError) + fs.SetOutput(a.err) + id := fs.String("id", "", "Runbook id") + name := fs.String("name", "", "Runbook name") + desc := fs.String("desc", "", "Description") + edit := fs.Bool("edit", false, "Open interactive template in $EDITOR") + from := fs.String("from", "", "Read runbook JSON from file") + step := multiStringFlag{} + fs.Var(&step, "step", "Step in format 'Title|command'") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + var rbInput cluster.Runbook + fromPath := strings.TrimSpace(*from) + if fromPath != "" { + raw, err := os.ReadFile(fromPath) + if err != nil { + fmt.Fprintf(a.err, "runbook add: read --from: %v\n", err) + return 1 + } + parsed, err := parseRunbookJSON(raw) + if err != nil { + fmt.Fprintf(a.err, "runbook add: invalid --from json: %v\n", err) + return 1 + } + rbInput = parsed + } else if *edit { + input := cluster.Runbook{ + ID: strings.TrimSpace(*id), + Name: strings.TrimSpace(*name), + Description: strings.TrimSpace(*desc), + Steps: []cluster.RunbookStep{ + {Title: "Check agent", Command: "cluster agent status"}, + {Title: "Check drift", Command: "cluster drift"}, + }, + } + raw, _ := jsonMarshalIndent(input) + if isEmbeddedConsoleMode() { + path, err := writeTemplateFile("pxmon-runbook-*.json", raw) + if err != nil { + fmt.Fprintf(a.err, "runbook add: %v\n", err) + return 1 + } + fmt.Fprintf(a.out, "Template saved: %s\n", path) + fmt.Fprintf(a.out, "Edit it in external terminal and run:\n") + fmt.Fprintf(a.out, "pxmon cluster runbook add --from %s\n", shellQuoteArg(path)) + return 0 + } + edited, err := openEditorTemplate("pxmon-runbook-*.json", raw) + if err != nil { + fmt.Fprintf(a.err, "runbook add: %v\n", err) + return 1 + } + parsed, err := parseRunbookJSON(edited) + if err != nil { + fmt.Fprintf(a.err, "runbook add: invalid template: %v\n", err) + return 1 + } + rbInput = parsed + } else { + steps := make([]cluster.RunbookStep, 0, len(step)) + for _, raw := range step { + parts := strings.SplitN(raw, "|", 2) + title := strings.TrimSpace(parts[0]) + cmd := "" + if len(parts) > 1 { + cmd = strings.TrimSpace(parts[1]) + } + if title == "" { + continue + } + steps = append(steps, cluster.RunbookStep{Title: title, Command: cmd}) + } + rbInput = cluster.Runbook{ + ID: strings.TrimSpace(*id), + Name: strings.TrimSpace(*name), + Description: strings.TrimSpace(*desc), + Steps: steps, + } + } + rb, err := svc.AddRunbook(rbInput) + if err != nil { + fmt.Fprintf(a.err, "runbook add: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, rb) + return 0 + } + fmt.Fprintf(a.out, "Runbook added: %s (%s)\n", rb.Name, rb.ID) + return 0 + case "rm", "remove", "del": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster runbook rm ") + return 2 + } + rb, err := svc.RemoveRunbook(args[1]) + if err != nil { + fmt.Fprintf(a.err, "runbook rm: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, rb) + return 0 + } + fmt.Fprintf(a.out, "Runbook removed: %s (%s)\n", rb.Name, rb.ID) + return 0 + case "show": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster runbook show ") + return 2 + } + rb, ok := svc.GetRunbook(args[1]) + if !ok { + fmt.Fprintf(a.err, "runbook %q not found\n", args[1]) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, rb) + return 0 + } + fmt.Fprintf(a.out, "%s - %s\n", rb.ID, rb.Name) + if rb.Description != "" { + fmt.Fprintln(a.out, rb.Description) + } + for i, st := range rb.Steps { + fmt.Fprintf(a.out, "%d. %s\n", i+1, st.Title) + if st.Command != "" { + fmt.Fprintf(a.out, " %s\n", st.Command) + } + } + return 0 + case "run": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster runbook run ") + return 2 + } + rb, ok := svc.GetRunbook(args[1]) + if !ok { + fmt.Fprintf(a.err, "runbook %q not found\n", args[1]) + return 1 + } + type stepResult struct { + Title string `json:"title"` + Command string `json:"command"` + ExitCode int `json:"exit_code"` + Output string `json:"output,omitempty"` + } + results := make([]stepResult, 0, len(rb.Steps)) + for _, st := range rb.Steps { + if strings.TrimSpace(st.Command) == "" { + continue + } + out, code := runObserverScopedCommand(svc, svc.ConfigPath(), st.Command, observerCommandOptions{ + AllowShellEscape: false, + StatsAutoOnce: true, + BlockBotRun: true, + StripANSI: true, + }) + results = append(results, stepResult{Title: st.Title, Command: st.Command, ExitCode: code, Output: strings.TrimSpace(out)}) + } + _ = svc.AppendChange("runbook.run", rb.ID, rb.Name) + if jsonOut { + _ = writeJSON(a.out, map[string]any{"runbook": rb.ID, "results": results}) + for _, r := range results { + if r.ExitCode != 0 { + return 1 + } + } + return 0 + } + failed := false + for i, r := range results { + fmt.Fprintf(a.out, "%d. %s -> exit %d\n", i+1, r.Title, r.ExitCode) + if r.Output != "" { + fmt.Fprintln(a.out, r.Output) + } + if r.ExitCode != 0 { + failed = true + } + } + if failed { + return 1 + } + return 0 + default: + fmt.Fprintf(a.err, "unknown runbook subcommand %q\n", args[0]) + return 2 + } +} + +func (a *App) runClusterSchedule(svc *cluster.Service, args []string, jsonOut bool) int { + if len(args) == 0 { + fmt.Fprintln(a.err, "usage: pxmon cluster schedule ") + return 2 + } + sub := strings.ToLower(strings.TrimSpace(args[0])) + switch sub { + case "ls", "list", "show": + items, err := svc.SchedulerList() + if err != nil { + fmt.Fprintf(a.err, "schedule ls: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, items) + return 0 + } + if len(items) == 0 { + fmt.Fprintln(a.out, "(no tasks)") + return 0 + } + tw := tabwriter.NewWriter(a.out, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tNAME\tCLUSTER\tMODE\tEVERY\tBACKOFF\tRETRY\tENABLED\tNEXT_RUN\tCOMMAND") + for _, t := range items { + next := "-" + if !t.NextRunAt.IsZero() { + next = t.NextRunAt.Format("2006-01-02 15:04:05") + } + clusterSel := t.Cluster + if strings.TrimSpace(clusterSel) == "" { + clusterSel = "(active)" + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%d/%d\t%t\t%s\t%s\n", + t.ID, t.Name, clusterSel, t.Mode, t.Every, t.Backoff, t.RetryCur, t.RetryMax, t.Enabled, next, t.Command) + } + _ = tw.Flush() + return 0 + case "add": + fs := flag.NewFlagSet("cluster schedule add", flag.ContinueOnError) + fs.SetOutput(a.err) + name := fs.String("name", "", "Task name") + cmd := fs.String("cmd", "", "Command to execute") + clusterSel := fs.String("cluster", "", "Cluster selector (default: active cluster)") + mode := fs.String("mode", "shell", "Execution mode: shell|observer") + every := fs.String("every", "5m", "Run interval duration, e.g. 5m, 1h") + backoff := fs.String("backoff", "30s", "Retry backoff base, e.g. 30s") + jitter := fs.Int("jitter-sec", 5, "Retry jitter in seconds") + retryMax := fs.Int("retry-max", 3, "Maximum retries before next regular interval") + enabled := fs.Bool("enabled", true, "Enable task immediately") + edit := fs.Bool("edit", false, "Open interactive template in $EDITOR") + from := fs.String("from", "", "Read task JSON from file") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + addName := strings.TrimSpace(*name) + addCluster := strings.TrimSpace(*clusterSel) + addCmd := strings.TrimSpace(*cmd) + addEvery := strings.TrimSpace(*every) + addMode := strings.TrimSpace(*mode) + addBackoff := strings.TrimSpace(*backoff) + addJitter := *jitter + addRetryMax := *retryMax + addEnabled := *enabled + fromPath := strings.TrimSpace(*from) + if fromPath != "" { + raw, err := os.ReadFile(fromPath) + if err != nil { + fmt.Fprintf(a.err, "schedule add: read --from: %v\n", err) + return 1 + } + parsed, err := parseScheduleJSON(raw) + if err != nil { + fmt.Fprintf(a.err, "schedule add: invalid --from json: %v\n", err) + return 1 + } + addName = strings.TrimSpace(parsed.Name) + addCluster = strings.TrimSpace(parsed.Cluster) + addCmd = strings.TrimSpace(parsed.Command) + addEvery = strings.TrimSpace(parsed.Every) + addMode = strings.TrimSpace(parsed.Mode) + addBackoff = strings.TrimSpace(parsed.Backoff) + addJitter = parsed.JitterSec + addRetryMax = parsed.RetryMax + addEnabled = parsed.Enabled + } else if *edit { + input := cluster.ScheduledTask{ + Name: addName, + Cluster: addCluster, + Command: addCmd, + Mode: addMode, + Every: addEvery, + Backoff: addBackoff, + JitterSec: addJitter, + RetryMax: addRetryMax, + Enabled: addEnabled, + } + if strings.TrimSpace(input.Name) == "" { + input.Name = "new-task" + } + if strings.TrimSpace(input.Command) == "" { + input.Command = "cluster drift" + } + if strings.TrimSpace(input.Mode) == "" { + input.Mode = "observer" + } + if strings.TrimSpace(input.Every) == "" { + input.Every = "30m" + } + if strings.TrimSpace(input.Backoff) == "" { + input.Backoff = "30s" + } + if input.JitterSec <= 0 { + input.JitterSec = 5 + } + if input.RetryMax <= 0 { + input.RetryMax = 3 + } + raw, _ := jsonMarshalIndent(input) + if isEmbeddedConsoleMode() { + path, err := writeTemplateFile("pxmon-schedule-*.json", raw) + if err != nil { + fmt.Fprintf(a.err, "schedule add: %v\n", err) + return 1 + } + fmt.Fprintf(a.out, "Template saved: %s\n", path) + fmt.Fprintf(a.out, "Edit it in external terminal and run:\n") + fmt.Fprintf(a.out, "pxmon cluster schedule add --from %s\n", shellQuoteArg(path)) + return 0 + } + edited, err := openEditorTemplate("pxmon-schedule-*.json", raw) + if err != nil { + fmt.Fprintf(a.err, "schedule add: %v\n", err) + return 1 + } + parsed, err := parseScheduleJSON(edited) + if err != nil { + fmt.Fprintf(a.err, "schedule add: invalid template: %v\n", err) + return 1 + } + addName = strings.TrimSpace(parsed.Name) + addCluster = strings.TrimSpace(parsed.Cluster) + addCmd = strings.TrimSpace(parsed.Command) + addEvery = strings.TrimSpace(parsed.Every) + addMode = strings.TrimSpace(parsed.Mode) + addBackoff = strings.TrimSpace(parsed.Backoff) + addJitter = parsed.JitterSec + addRetryMax = parsed.RetryMax + addEnabled = parsed.Enabled + } + t, err := svc.SchedulerAdd(addName, addCluster, addCmd, addEvery, addMode, addBackoff, addJitter, addRetryMax, addEnabled) + if err != nil { + fmt.Fprintf(a.err, "schedule add: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, t) + return 0 + } + fmt.Fprintf(a.out, "Task added: %s (%s)\n", t.Name, t.ID) + return 0 + case "rm", "remove", "del": + if len(args) < 2 { + fmt.Fprintln(a.err, "usage: pxmon cluster schedule rm ") + return 2 + } + t, err := svc.SchedulerRemove(args[1]) + if err != nil { + fmt.Fprintf(a.err, "schedule rm: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, t) + return 0 + } + fmt.Fprintf(a.out, "Task removed: %s\n", t.Name) + return 0 + case "run-due": + due, err := svc.SchedulerDue(time.Now().UTC()) + if err != nil { + fmt.Fprintf(a.err, "schedule run-due: %v\n", err) + return 1 + } + type taskRun struct { + ID string `json:"id"` + Name string `json:"name"` + ExitCode int `json:"exit_code"` + Output string `json:"output,omitempty"` + } + results := make([]taskRun, 0, len(due)) + failed := false + for _, t := range due { + out, code := runScheduledTaskNow(svc, t) + results = append(results, taskRun{ID: t.ID, Name: t.Name, ExitCode: code, Output: strings.TrimSpace(out)}) + _ = svc.SchedulerMarkResult(t.ID, code == 0, time.Now().UTC()) + if code != 0 { + failed = true + } + } + _ = svc.AppendChange("scheduler.run-due", "tasks", fmt.Sprintf("count=%d", len(results))) + if jsonOut { + _ = writeJSON(a.out, map[string]any{"ran": len(results), "results": results}) + if failed { + return 1 + } + return 0 + } + if len(results) == 0 { + fmt.Fprintln(a.out, "No due tasks") + return 0 + } + for _, r := range results { + fmt.Fprintf(a.out, "%s (%s) -> exit %d\n", r.Name, r.ID, r.ExitCode) + if r.Output != "" { + fmt.Fprintln(a.out, r.Output) + } + } + if failed { + return 1 + } + return 0 + case "start": + fs := flag.NewFlagSet("cluster schedule start", flag.ContinueOnError) + fs.SetOutput(a.err) + interval := fs.Duration("interval", 30*time.Second, "Worker polling interval") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + pid, logPath, err := startSchedulerDaemon(svc, *interval) + if err != nil { + fmt.Fprintf(a.err, "schedule start: %v\n", err) + return 1 + } + fmt.Fprintf(a.out, "Scheduler started (pid %d)\n", pid) + fmt.Fprintf(a.out, "Log: %s\n", logPath) + return 0 + case "stop": + if err := stopSchedulerDaemon(svc); err != nil { + fmt.Fprintf(a.err, "schedule stop: %v\n", err) + return 1 + } + fmt.Fprintln(a.out, "Scheduler stopped") + return 0 + case "status": + pid, running, logPath, err := schedulerDaemonStatus(svc) + if err != nil { + fmt.Fprintf(a.err, "schedule status: %v\n", err) + return 1 + } + if jsonOut { + _ = writeJSON(a.out, map[string]any{"pid": pid, "running": running, "log": logPath}) + return 0 + } + fmt.Fprintf(a.out, "running: %t\n", running) + if pid > 0 { + fmt.Fprintf(a.out, "pid: %d\n", pid) + } + fmt.Fprintf(a.out, "log: %s\n", logPath) + return 0 + case "logs": + fs := flag.NewFlagSet("cluster schedule logs", flag.ContinueOnError) + fs.SetOutput(a.err) + tail := fs.Int("tail", 200, "Last N lines") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + _, logPath := schedulerDaemonPaths(svc) + lines, err := readLastLines(logPath, *tail) + if err != nil { + fmt.Fprintf(a.err, "schedule logs: %v\n", err) + return 1 + } + for _, ln := range lines { + fmt.Fprintln(a.out, ln) + } + return 0 + case "worker": + fs := flag.NewFlagSet("cluster schedule worker", flag.ContinueOnError) + fs.SetOutput(a.err) + interval := fs.Duration("interval", 30*time.Second, "Polling interval") + if err := fs.Parse(args[1:]); err != nil { + return 2 + } + return runSchedulerWorkerLoop(svc, *interval, a.err) + default: + fmt.Fprintf(a.err, "unknown schedule subcommand %q\n", args[0]) + return 2 + } +} + +func normalizeCLIPathList(in []string) []string { + out := make([]string, 0, len(in)) + seen := map[string]struct{}{} + for _, raw := range in { + for _, p := range strings.Split(raw, ",") { + v := strings.TrimSpace(p) + if v == "" { + continue + } + if _, ok := seen[v]; ok { + continue + } + seen[v] = struct{}{} + out = append(out, v) + } + } + return out +} + +func isEmbeddedConsoleMode() bool { + return strings.TrimSpace(os.Getenv("PXMON_EMBEDDED_CONSOLE")) == "1" +} + +func writeTemplateFile(pattern string, initial []byte) (string, error) { + f, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + defer f.Close() + if len(initial) == 0 { + initial = []byte("{}\n") + } + if _, err := f.Write(initial); err != nil { + return "", err + } + return f.Name(), nil +} + +func openEditorTemplate(pattern string, initial []byte) ([]byte, error) { + editor := strings.TrimSpace(os.Getenv("VISUAL")) + if editor == "" { + editor = strings.TrimSpace(os.Getenv("EDITOR")) + } + if editor == "" { + editor = "vi" + } + f, err := os.CreateTemp("", pattern) + if err != nil { + return nil, err + } + tmpPath := f.Name() + defer os.Remove(tmpPath) + if len(initial) == 0 { + initial = []byte("{}\n") + } + if _, err := f.Write(initial); err != nil { + _ = f.Close() + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + cmd := exec.Command("sh", "-lc", shellQuoteArg(editor)+" "+shellQuoteArg(tmpPath)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("open editor %q: %w", editor, err) + } + b, err := os.ReadFile(tmpPath) + if err != nil { + return nil, err + } + if strings.TrimSpace(string(b)) == "" { + return nil, fmt.Errorf("template is empty") + } + return b, nil +} + +func parseRunbookJSON(raw []byte) (cluster.Runbook, error) { + var rb cluster.Runbook + if err := json.Unmarshal(raw, &rb); err != nil { + return cluster.Runbook{}, err + } + rb.ID = strings.TrimSpace(rb.ID) + rb.Name = strings.TrimSpace(rb.Name) + rb.Description = strings.TrimSpace(rb.Description) + outSteps := make([]cluster.RunbookStep, 0, len(rb.Steps)) + for _, st := range rb.Steps { + title := strings.TrimSpace(st.Title) + cmd := strings.TrimSpace(st.Command) + note := strings.TrimSpace(st.Note) + if title == "" { + continue + } + outSteps = append(outSteps, cluster.RunbookStep{Title: title, Command: cmd, Note: note}) + } + rb.Steps = outSteps + return rb, nil +} + +func parseScheduleJSON(raw []byte) (cluster.ScheduledTask, error) { + var t cluster.ScheduledTask + if err := json.Unmarshal(raw, &t); err != nil { + return cluster.ScheduledTask{}, err + } + t.Name = strings.TrimSpace(t.Name) + t.Cluster = strings.TrimSpace(t.Cluster) + t.Command = strings.TrimSpace(t.Command) + t.Mode = strings.ToLower(strings.TrimSpace(t.Mode)) + t.Every = strings.TrimSpace(t.Every) + t.Backoff = strings.TrimSpace(t.Backoff) + if t.JitterSec < 0 { + t.JitterSec = 0 + } + if t.RetryMax <= 0 { + t.RetryMax = 3 + } + return t, nil +} + +func shellQuoteArg(v string) string { + if v == "" { + return "''" + } + return "'" + strings.ReplaceAll(v, "'", `'\''`) + "'" +} + +func runScheduledTaskNow(svc *cluster.Service, t cluster.ScheduledTask) (string, int) { + switch strings.ToLower(strings.TrimSpace(t.Mode)) { + case "observer": + return runObserverScopedCommand(svc, svc.ConfigPath(), t.Command, observerCommandOptions{ + AllowShellEscape: false, + StatsAutoOnce: true, + BlockBotRun: true, + StripANSI: true, + }) + default: + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + out, err := svc.RunRemoteShell(ctx, t.Cluster, t.Command) + if err != nil { + return err.Error(), 1 + } + return out, 0 + } +} + +func schedulerDaemonPaths(svc *cluster.Service) (string, string) { + base := filepath.Join(svc.DataDir(), "scheduler") + return filepath.Join(base, "daemon.pid"), filepath.Join(base, "daemon.log") +} + +func startSchedulerDaemon(svc *cluster.Service, interval time.Duration) (int, string, error) { + if interval < 5*time.Second { + interval = 5 * time.Second + } + if err := stopSchedulerDaemon(svc); err != nil { + _ = err + } + pidPath, logPath := schedulerDaemonPaths(svc) + if err := os.MkdirAll(filepath.Dir(pidPath), 0o700); err != nil { + return 0, "", err + } + logf, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return 0, "", err + } + defer logf.Close() + exe, err := os.Executable() + if err != nil { + return 0, "", err + } + args := []string{"cluster", "schedule", "worker", "--interval", interval.String()} + cmd := exec.Command(exe, args...) + cmd.Stdout = logf + cmd.Stderr = logf + cmd.Stdin = nil + if err := cmd.Start(); err != nil { + return 0, "", err + } + pid := cmd.Process.Pid + _ = os.WriteFile(pidPath, []byte(fmt.Sprintf("%d\n", pid)), 0o600) + return pid, logPath, nil +} + +func stopSchedulerDaemon(svc *cluster.Service) error { + pidPath, _ := schedulerDaemonPaths(svc) + raw, err := os.ReadFile(pidPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + pid, _ := strconv.Atoi(strings.TrimSpace(string(raw))) + if pid > 0 { + proc, err := os.FindProcess(pid) + if err == nil { + _ = proc.Kill() + } + } + _ = os.Remove(pidPath) + return nil +} + +func schedulerDaemonStatus(svc *cluster.Service) (int, bool, string, error) { + pidPath, logPath := schedulerDaemonPaths(svc) + raw, err := os.ReadFile(pidPath) + if err != nil { + if os.IsNotExist(err) { + return 0, false, logPath, nil + } + return 0, false, logPath, err + } + pid, _ := strconv.Atoi(strings.TrimSpace(string(raw))) + if pid <= 0 { + return 0, false, logPath, nil + } + if err := exec.Command("ps", "-p", strconv.Itoa(pid), "-o", "pid=").Run(); err != nil { + return pid, false, logPath, nil + } + return pid, true, logPath, nil +} + +func runSchedulerWorkerLoop(svc *cluster.Service, interval time.Duration, w io.Writer) int { + if interval < 5*time.Second { + interval = 5 * time.Second + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + due, err := svc.SchedulerDue(time.Now().UTC()) + if err != nil { + fmt.Fprintf(w, "scheduler worker: list due: %v\n", err) + } else { + for _, t := range due { + out, code := runScheduledTaskNow(svc, t) + if code != 0 { + fmt.Fprintf(w, "scheduler worker: task %s failed (%d): %s\n", t.Name, code, strings.TrimSpace(out)) + } else if strings.TrimSpace(out) != "" { + fmt.Fprintf(w, "scheduler worker: task %s ok: %s\n", t.Name, strings.TrimSpace(out)) + } else { + fmt.Fprintf(w, "scheduler worker: task %s ok\n", t.Name) + } + _ = svc.SchedulerMarkResult(t.ID, code == 0, time.Now().UTC()) + } + } + <-ticker.C + } +} diff --git a/internal/cli/monitor.go b/internal/cli/monitor.go new file mode 100644 index 0000000..1f0eba9 --- /dev/null +++ b/internal/cli/monitor.go @@ -0,0 +1,5508 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "text/tabwriter" + "time" + "unicode/utf8" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/creack/pty" + "github.com/hinshun/vt10x" + "golang.org/x/term" + + "pxmon/internal/agent" + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +type MonitorOptions struct { + Interval time.Duration + InitialIface string + AlertPolicy cluster.AlertPolicy + InitialView string // "overview" | "clusters" | "network" +} + +type ifaceRate struct { + RxMbps float64 + TxMbps float64 +} + +type monitorRuntime struct { + policy cluster.AlertPolicy + selectedIface string + prevAt time.Time + prevNet map[string]agent.NetworkStat + rates map[string]ifaceRate + vmWarnings []string + vmLastCheck time.Time +} + +type monitorStatsMsg struct { + stats agent.StatsResponse + err error +} + +type monitorTickMsg struct{} +type spinnerTickMsg struct{} +type vmAlertMsg struct { + warnings []string + err error +} + +type terminalStartedMsg struct { + session *terminalSession + err error +} + +type terminalOutputMsg struct { + data string + err error +} + +type terminalClosedMsg struct { + err error +} + +type historyLoadedMsg struct { + points map[string][]ifaceHistoryPoint + count int + err error +} + +type historyAppendMsg struct { + err error +} + +type consoleExecResultMsg struct { + ID int64 + Command string + Output string + ExitCode int +} + +type telegramSyncMsg struct { + updated cluster.Telegram + err error +} + +type monitorView int + +const ( + viewOverview monitorView = iota + viewClusters + viewNetwork + viewSettings + viewDocs + viewUsage + viewLive +) + +type inputMode int + +const ( + inputNone inputMode = iota + inputSearch + inputLockerNew + inputLockerConfirm +) + +type historyRange struct { + Label string + Since time.Time + All bool +} + +type tuiSettings struct { + RefreshMillis int `json:"refresh_millis"` + PageSize int `json:"page_size"` + HistoryRange string `json:"history_range"` +} + +type terminalSession struct { + cmd *exec.Cmd + ptmx *os.File + shell string +} + +type graphMetric int + +const ( + metricRX graphMetric = iota + metricTX + metricTotal +) + +type ifaceHistoryPoint struct { + At time.Time + RxMbps float64 + TxMbps float64 + RxDrops uint64 + TxDrops uint64 +} + +type networkRow struct { + Interface string + CurRxMbps float64 + CurTxMbps float64 + CurTotalMbps float64 + AvgTotalMbps float64 + PeakTotal float64 + ConsumedByte uint64 + Samples int + RxDrops uint64 + TxDrops uint64 + Spark string +} + +type monitorModel struct { + svc *cluster.Service + cluster cluster.Cluster + hasCluster bool + interval time.Duration + runtime monitorRuntime + history map[string][]ifaceHistoryPoint + historyStore *history.NetworkStore + historyRange historyRange + + view monitorView + inputMode inputMode + inputBuf string + searchTerm string + statusMsg string + termMode bool + termFull bool + termReady bool + termErr string + term *terminalSession + termLines []string + termPartial string + termCursor int + termHistory []string + termHistPos int + termDraft string + termBusy bool + thinkFrame int + spinnerActive bool + spinnerStart time.Time + contentScroll int + termScroll int + termCont string + termLastSubmit string + termLastSubmitAt time.Time + termExecSeq int64 + termExecActive int64 + settingsPath string + settingsCursor int + docsScroll int + telegram cluster.Telegram + telegramSyncing bool + lockerEnabled bool + lockerHash string + lockInput string + lockErr string + locked bool + lastUnlockAt time.Time + lockerPending string + + page int + pageSize int + cursor int + metric graphMetric + pinnedIface string // if non-empty, cursor follows this interface across refreshes + + usageRange history.RangeShortcut + usageSnap *cluster.UsageSnapshot + usageLoading bool + usageErr error + + stats agent.StatsResponse + hasStats bool + lastErr error + loading bool + width int + height int + + sshMode bool + sshClosed bool + sshErr string + sshSess *cluster.InteractiveSession + sshCluster cluster.Cluster + sshVT vt10x.Terminal + sshCols int + sshRows int + sshPendingRepaint bool + sshScrollback []string + sshScrollPending string + sshScrollOffset int + + privacyMode bool + + liveEntries []liveClusterStat + liveLoading bool + liveError string + livePage int + liveSort int + livePinned map[string]bool + liveCursor int + + liveCmdActive bool + liveCmdSpec livePluginSpec + liveCmdBuffer string + liveCmdErr string + liveCmdRunning bool + liveCmdLastRun time.Time + liveCmdInterval time.Duration + liveCmdScroll int +} + +var ( + ccAccent = lipgloss.Color("#D97757") + ccAccentDim = lipgloss.Color("#A55A3F") + ccBorder = lipgloss.Color("#3A3A3A") + ccMuted = lipgloss.Color("#7A7A7A") + ccMutedSoft = lipgloss.Color("#5A5A5A") + ccText = lipgloss.Color("#E6E6E6") + ccTextBright = lipgloss.Color("#FFFFFF") + ccSuccess = lipgloss.Color("#7FB069") + ccInfo = lipgloss.Color("#6FA8DC") + ccWarn = lipgloss.Color("#E6B450") + ccCrit = lipgloss.Color("#E06C75") + + thinBorder = lipgloss.RoundedBorder() + + rootStyle = lipgloss.NewStyle().Foreground(ccText) + + panelStyle = lipgloss.NewStyle(). + BorderStyle(thinBorder). + BorderForeground(ccBorder). + Padding(0, 1) + + headerPanelStyle = panelStyle.Copy().BorderForeground(ccAccent) + commandPanelStyle = panelStyle.Copy().BorderForeground(ccAccent) + + titleStyle = lipgloss.NewStyle().Bold(true).Foreground(ccTextBright) + dimStyle = lipgloss.NewStyle().Foreground(ccMuted) + warnStyle = lipgloss.NewStyle().Bold(true).Foreground(ccWarn) + critStyle = lipgloss.NewStyle().Bold(true).Foreground(ccCrit) + okStyle = lipgloss.NewStyle().Bold(true).Foreground(ccSuccess) + accentStyle = lipgloss.NewStyle().Foreground(ccAccent) + brightStyle = lipgloss.NewStyle().Bold(true).Foreground(ccTextBright) + softStyle = lipgloss.NewStyle().Foreground(ccMutedSoft) + + ansiCSIRegex = regexp.MustCompile(`\x1b\[[0-9;?]*[ -/]*[@-~]`) + ansiOSCRegex = regexp.MustCompile(`\x1b\][^\a]*(\a|\x1b\\)`) +) + +func (a *App) runClusterMonitor(svc *cluster.Service, selector string, opts MonitorOptions) int { + c := cluster.Cluster{} + hasCluster := true + resolved, err := svc.Get(selector) + if err != nil { + if errorsIsNoActive(err) && strings.TrimSpace(selector) == "" { + hasCluster = false + c = cluster.Cluster{Name: "(none)"} + } else { + if errorsIsNoActive(err) { + fmt.Fprintln(a.err, "no active cluster") + return 1 + } + fmt.Fprintf(a.err, "resolve cluster: %v\n", err) + return 1 + } + } else { + c = resolved + } + + if opts.Interval <= 0 { + opts.Interval = 2 * time.Second + } + if opts.Interval < 500*time.Millisecond { + opts.Interval = 500 * time.Millisecond + } + + settingsPath := filepath.Join(svc.DataDir(), "tui_settings.json") + stored, _ := loadTUISettings(settingsPath) + if stored.RefreshMillis >= 500 { + opts.Interval = time.Duration(stored.RefreshMillis) * time.Millisecond + } + pageSize := 8 + if stored.PageSize >= 5 && stored.PageSize <= 50 { + pageSize = stored.PageSize + } + range30d := defaultHistoryRange() + if strings.TrimSpace(stored.HistoryRange) != "" { + if parsed, parseErr := parseHistoryRange(stored.HistoryRange); parseErr == nil { + range30d = parsed + } + } + lockerCfg, _ := svc.GetLocker() + lockerEnabled := lockerCfg.Enabled && strings.TrimSpace(lockerCfg.PasswordHash) != "" + locked := false + if lockerEnabled { + if isLocked, _, lockErr := svc.IsLocked(); lockErr == nil { + locked = isLocked + } else { + locked = true + } + } + tgCfg, _ := svc.GetTelegram() + + stdinFD := int(os.Stdin.Fd()) + stdoutFD := int(os.Stdout.Fd()) + interactive := term.IsTerminal(stdinFD) && term.IsTerminal(stdoutFD) + if !interactive { + if !hasCluster { + fmt.Fprintln(a.err, "no active cluster (connect one first, e.g. pxmon cluster connect --name ...)") + return 1 + } + stats, fetchErr := fetchAgentStats(svc, c.Name) + if fetchErr != nil { + fmt.Fprintf(a.err, "cluster stats: %v\n", fetchErr) + return 1 + } + printStatsSnapshot(a.out, stats) + return 0 + } + + view := parseView(opts.InitialView) + + model := monitorModel{ + svc: svc, + cluster: c, + interval: opts.Interval, + runtime: monitorRuntime{ + policy: opts.AlertPolicy, + selectedIface: strings.TrimSpace(opts.InitialIface), + prevNet: map[string]agent.NetworkStat{}, + rates: map[string]ifaceRate{}, + }, + history: map[string][]ifaceHistoryPoint{}, + historyStore: history.NewNetworkStore(svc.DataDir()), + historyRange: range30d, + settingsPath: settingsPath, + telegram: tgCfg, + lockerEnabled: lockerEnabled, + lockerHash: strings.TrimSpace(lockerCfg.PasswordHash), + locked: locked, + + view: view, + page: 1, + pageSize: pageSize, + cursor: 0, + metric: metricTotal, + termLines: make([]string, 0, 512), + termHistory: make([]string, 0, 128), + termHistPos: -1, + + loading: true, + width: 120, + height: 40, + } + model.hasCluster = hasCluster + + prog := tea.NewProgram(model, tea.WithAltScreen()) + finalModel, err := prog.Run() + if err != nil { + fmt.Fprintf(a.err, "cluster stats tui: %v\n", err) + return 1 + } + if m, ok := finalModel.(monitorModel); ok { + if m.term != nil { + m.term.close() + } + } + return 0 +} + +func (m monitorModel) Init() tea.Cmd { + if !m.hasCluster { + return nil + } + return tea.Batch( + m.fetchStatsCmd(), + m.loadHistoryCmd(), + ) +} + +func (m monitorModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch v := msg.(type) { + case tea.WindowSizeMsg: + m.width = v.Width + m.height = v.Height + if m.term != nil && m.term.ptmx != nil { + _ = pty.Setsize(m.term.ptmx, &pty.Winsize{ + Cols: uint16(max(40, m.width-4)), + Rows: uint16(max(8, m.terminalRows()-2)), + }) + } + if m.sshMode { + m.handleSSHResize() + } + return m, nil + case tea.KeyMsg: + if m.locked { + return m.handleLockKey(v) + } + return m.handleKey(v) + case terminalStartedMsg: + if v.err != nil { + m.termErr = v.err.Error() + m.setStatus("terminal start failed") + m.termMode = false + return m, nil + } + m.term = v.session + m.termReady = true + m.termErr = "" + m.setStatus("terminal ready") + return m, m.readTerminalCmd() + case terminalOutputMsg: + if v.data != "" { + m.appendTerminalText(v.data) + } + if v.err != nil { + m.termErr = v.err.Error() + m.termReady = false + return m, nil + } + return m, m.readTerminalCmd() + case terminalClosedMsg: + if v.err != nil { + m.termErr = v.err.Error() + m.setStatus("terminal closed with error") + } else { + m.setStatus("terminal closed") + } + m.termReady = false + if m.term != nil { + m.term.close() + } + m.term = nil + m.termMode = false + m.termFull = false + return m, nil + case sshStartedMsg: + if v.err != nil { + m.setStatus("openssh: " + v.err.Error()) + m.appendConsoleOutput("openssh: " + v.err.Error()) + return m, nil + } + m.enterEmbeddedSSH(v) + m.setStatus(fmt.Sprintf("ssh connected: %s", v.cluster.Name)) + return m, readSSHChunkCmd(m.sshSess) + case sshChunkMsg: + if !m.sshMode || m.sshSess == nil { + return m, nil + } + if m.sshVT != nil && len(v.data) > 0 { + _, _ = m.sshVT.Write(v.data) + m.captureSSHScrollback(v.data) + } + if v.err != nil { + msg := v.err.Error() + name := m.sshCluster.Name + m.exitEmbeddedSSH(fmt.Sprintf("ssh closed (%s): %s", name, msg)) + m.appendConsoleOutput(fmt.Sprintf("ssh session closed (%s): %s", name, msg)) + return m, nil + } + return m, readSSHChunkCmd(m.sshSess) + case sshClosedMsg: + reason := "" + if v.err != nil { + reason = v.err.Error() + } + m.exitEmbeddedSSH(fmt.Sprintf("ssh closed (%s)%s", m.sshCluster.Name, func() string { + if reason != "" { + return ": " + reason + } + return "" + }())) + return m, nil + case sshTickMsg: + return m, nil + case monitorLiveMsg: + m.liveLoading = false + if v.err != nil { + m.liveError = v.err.Error() + m.setStatus("live error: " + v.err.Error()) + } else { + m.liveError = "" + m.liveEntries = v.entries + if m.liveCursor >= len(m.liveEntries) { + m.liveCursor = len(m.liveEntries) - 1 + } + if m.liveCursor < 0 { + m.liveCursor = 0 + } + m.setStatus(fmt.Sprintf("live: %d clusters", len(v.entries))) + } + if m.view == viewLive { + return m, liveAutoTickCmd() + } + return m, nil + case liveAutoTickMsg: + if m.view == viewLive { + m.liveLoading = true + return m, m.fetchLiveCmd() + } + return m, nil + case liveCmdResultMsg: + if !m.liveCmdActive { + return m, nil + } + m.liveCmdRunning = false + m.liveCmdLastRun = time.Now() + if v.Err != "" { + m.liveCmdErr = v.Err + } else { + m.liveCmdErr = "" + m.liveCmdBuffer = v.Output + } + return m, liveCmdTickCmd(m.liveCmdInterval) + case liveCmdTickMsg: + if !m.liveCmdActive { + return m, nil + } + m.liveCmdRunning = true + return m, m.runLiveCmdCmd(m.liveCmdSpec) + case monitorUsageMsg: + m.usageLoading = false + m.usageErr = v.err + if v.err == nil { + snap := v.snap + m.usageSnap = &snap + m.setStatus("usage: " + m.usageRange.Label()) + } else { + m.setStatus("usage error: " + v.err.Error()) + } + return m, nil + case monitorStatsMsg: + m.loading = false + m.lastErr = v.err + + cmds := []tea.Cmd{m.nextTickCmd()} + if v.err == nil { + m.hasStats = true + m.stats = v.stats + m.runtime.update(v.stats) + + snap := m.makeHistorySnapshot(v.stats) + if len(snap.Interfaces) > 0 { + m.applyHistorySnapshot(snap) + cmds = append(cmds, m.appendHistoryCmd(snap)) + } + if time.Since(m.runtime.vmLastCheck) > 30*time.Second { + cmds = append(cmds, m.fetchVMAlertCmd()) + } + } + + m.clampNetworkNav() + m.snapToPinnedIface() + return m, tea.Batch(cmds...) + case vmAlertMsg: + m.runtime.vmLastCheck = time.Now() + if v.err != nil { + m.runtime.vmWarnings = nil + return m, nil + } + m.runtime.vmWarnings = append([]string(nil), v.warnings...) + return m, nil + case monitorTickMsg: + if m.lockerEnabled { + isLocked, _, err := m.svc.IsLocked() + if err != nil { + m.locked = true + m.lockInput = "" + m.lockErr = "locker check failed: " + err.Error() + return m, nil + } + if isLocked { + m.locked = true + m.lockInput = "" + m.lockErr = "session locked: enter password" + return m, nil + } + } + if !m.hasCluster { + return m, nil + } + if m.loading { + return m, m.nextTickCmd() + } + m.loading = true + return m, m.fetchStatsCmd() + case spinnerTickMsg: + if !m.termBusy { + m.spinnerActive = false + return m, nil + } + m.thinkFrame++ + return m, spinnerTickCmd() + case historyLoadedMsg: + if v.err != nil { + m.setStatus("history load failed: " + v.err.Error()) + return m, nil + } + m.history = v.points + m.setStatus(fmt.Sprintf("history loaded: %d snapshots (%s)", v.count, m.historyRange.Label)) + m.clampNetworkNav() + m.snapToPinnedIface() + return m, nil + case historyAppendMsg: + if v.err != nil { + m.setStatus("history append failed: " + v.err.Error()) + } + return m, nil + case consoleExecResultMsg: + if m.termExecActive != 0 && v.ID != 0 && v.ID != m.termExecActive { + return m, nil + } + m.termExecActive = 0 + m.termBusy = false + m.spinnerActive = false + if strings.TrimSpace(v.Output) != "" { + m.appendConsoleOutput(v.Output) + } + if v.ExitCode == 0 { + m.setStatus("ok: " + v.Command) + } else { + m.setStatus(fmt.Sprintf("failed (%d): %s", v.ExitCode, v.Command)) + } + + cmds := []tea.Cmd{} + if tg, tgErr := m.svc.GetTelegram(); tgErr == nil { + m.telegram = tg + } + if lk, lkErr := m.svc.GetLocker(); lkErr == nil { + m.lockerEnabled = lk.Enabled + m.lockerHash = strings.TrimSpace(lk.PasswordHash) + } + if current, err := m.svc.Get(""); err == nil { + if !m.hasCluster || current.ID != m.cluster.ID { + m.cluster = current + m.hasCluster = true + m.hasStats = false + m.runtime.prevAt = time.Time{} + m.runtime.prevNet = map[string]agent.NetworkStat{} + m.runtime.rates = map[string]ifaceRate{} + m.runtime.vmWarnings = nil + m.runtime.vmLastCheck = time.Time{} + m.history = map[string][]ifaceHistoryPoint{} + m.page = 1 + m.cursor = 0 + m.pinnedIface = "" + m.setStatus("active cluster: " + current.Name) + } + if !m.loading { + m.loading = true + cmds = append(cmds, m.fetchStatsCmd()) + } + cmds = append(cmds, m.loadHistoryCmd()) + } else if errorsIsNoActive(err) { + m.hasCluster = false + m.cluster = cluster.Cluster{Name: "(none)"} + m.hasStats = false + m.history = map[string][]ifaceHistoryPoint{} + m.runtime.prevAt = time.Time{} + m.runtime.prevNet = map[string]agent.NetworkStat{} + m.runtime.rates = map[string]ifaceRate{} + m.runtime.vmWarnings = nil + m.runtime.vmLastCheck = time.Time{} + m.setStatus("no active cluster") + } + return m, tea.Batch(cmds...) + case telegramSyncMsg: + m.telegramSyncing = false + if v.err != nil { + m.setStatus("telegram update failed: " + v.err.Error()) + return m, nil + } + m.telegram = v.updated + m.setStatus("telegram bot: " + ternary(v.updated.Enabled, "enabled", "disabled")) + return m, nil + } + return m, nil +} + +func (m monitorModel) View() string { + width := m.width - 2 + if width <= 0 { + width = 118 + } + if m.liveCmdActive { + height := m.height + if height <= 0 { + height = 32 + } + return m.applyPrivacy(m.renderLiveCmdView(width, height)) + } + if m.sshMode { + return m.applyPrivacy(m.renderSSHView()) + } + if width < 68 { + return dimStyle.Render("Terminal width is too small for dashboard. Increase width to >= 70 columns.") + } + if m.termMode && m.termFull { + sections := []string{ + m.renderHeader(width), + m.renderCommandDock(width), + } + return m.applyPrivacy(m.clampToTerminalHeight(rootStyle.Render(strings.Join(sections, "\n\n")))) + } + if m.locked { + return m.applyPrivacy(m.clampToTerminalHeight(m.renderLockedView(width))) + } + + header := m.renderHeader(width) + dock := m.renderCommandDock(width) + + // Compute how many lines are left for the main content panel so that the + // total view height matches the terminal height exactly. This prevents + // overflow into the terminal scrollback (which otherwise causes stale + // frames to stack up as the spinner ticks). + contentBudget := m.contentBudget(header, dock) + + content := "" + if !m.hasCluster { + switch m.view { + case viewDocs: + content = m.renderDocsDashboard(width) + case viewClusters: + content = m.renderClusterOverviewDashboard(width) + case viewSettings: + content = m.renderSettingsDashboard(width) + default: + panel, _ := renderFixedPanel(panelStyle, "Workspace", "No active cluster.\n\nOpen PXmon console with `t` and run:\ncluster connect --name eu-1 --host --user root --auth key --key-path ~/.ssh/id_ed25519", width, contentBudget, m.contentScroll) + content = panel + } + } else { + switch m.view { + case viewClusters: + content = m.renderClusterOverviewDashboard(width) + case viewNetwork: + content = m.renderNetworkDashboard(width) + case viewSettings: + content = m.renderSettingsDashboard(width) + case viewDocs: + content = m.renderDocsDashboard(width) + case viewUsage: + content = m.renderUsageDashboard(width) + case viewLive: + content = m.renderLiveDashboard(width) + default: + content = m.renderOverviewDashboard(width) + } + } + + if contentBudget > 0 { + clipped, _, _ := clipBodyToHeight(content, contentBudget, m.contentScroll) + content = clipped + } + + sections := []string{header, content, dock} + return m.applyPrivacy(m.clampToTerminalHeight(rootStyle.Render(strings.Join(sections, "\n\n")))) +} + +// contentBudget returns the number of lines available for the main content +// panel, given the already-rendered header and dock. Returns 0 when the +// terminal height is unknown or too small; callers should treat that as "do +// not clip". +func (m monitorModel) contentBudget(header, dock string) int { + if m.height <= 0 { + return 0 + } + // sections are joined with "\n\n" (2 separators between 3 panels = 2 blank + // lines). rootStyle.Render doesn't add extra trailing lines. + const separatorLines = 2 + budget := m.height - visibleHeight(header) - visibleHeight(dock) - separatorLines + if budget < 3 { + return 3 + } + return budget +} + +// clampToTerminalHeight forces the rendered output to have EXACTLY m.height +// lines: truncates if longer, pads with empty lines if shorter. A stable line +// count makes bubbletea's diff renderer behave predictably and prevents stale +// frames from accumulating in the terminal scrollback. +func (m monitorModel) clampToTerminalHeight(s string) string { + if m.height <= 0 { + return s + } + lines := strings.Split(s, "\n") + termWidth := m.width + if termWidth <= 0 { + termWidth = 120 + } + for i := range lines { + lines[i] = truncateVisible(lines[i], termWidth) + if pad := termWidth - visibleLen(lines[i]); pad > 0 { + lines[i] += strings.Repeat(" ", pad) + } + } + if len(lines) > m.height { + lines = lines[:m.height] + } + for len(lines) < m.height { + lines = append(lines, strings.Repeat(" ", termWidth)) + } + return strings.Join(lines, "\n") +} + +// applyPrivacy redacts sensitive patterns in the rendered view when the +// user has toggled privacy mode on. No-op otherwise. +func (m monitorModel) applyPrivacy(s string) string { + if !m.privacyMode { + return s + } + return applyPrivacyMultiline(s) +} + +func (m monitorModel) handleKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + if m.liveCmdActive { + return m.handleLiveCmdKey(v) + } + if m.sshMode { + return m.handleSSHKey(v) + } + if m.termMode { + return m.handleTerminalKey(v) + } + + if m.inputMode != inputNone { + return m.handleInputKey(v) + } + + switch v.String() { + case "q", "ctrl+c", "esc": + return m, tea.Quit + case "P", "alt+p": + m.privacyMode = !m.privacyMode + if m.privacyMode { + m.setStatus("privacy: on (alt+p to disable)") + } else { + m.setStatus("privacy: off") + } + return m, nil + case "t": + m.termMode = true + m.termFull = false + m.setStatus("pxmon console") + return m, nil + case "ctrl+t": + m.termMode = true + m.termFull = true + m.setStatus("pxmon console fullscreen") + return m, nil + case "s": + m.view = viewSettings + return m, nil + case "d", "D": + m.view = viewDocs + return m, nil + case "c": + m.view = viewClusters + return m, nil + case "u": + m.view = viewUsage + if m.usageRange == "" { + m.usageRange = history.RangeLive + } + m.usageLoading = true + m.setStatus("usage: loading") + return m, m.fetchUsageCmd(m.usageRange) + case "L": + m.view = viewLive + m.liveLoading = true + if m.livePinned == nil { + m.livePinned = map[string]bool{} + } + m.setStatus("live: loading") + return m, m.fetchLiveCmd() + case "/": + if m.view == viewNetwork { + m.inputMode = inputSearch + m.inputBuf = m.searchTerm + m.setStatus("search mode: type query, Enter apply, Esc cancel") + return m, nil + } + case "tab": + switch m.view { + case viewOverview: + m.view = viewClusters + case viewClusters: + m.view = viewNetwork + case viewNetwork: + m.view = viewSettings + case viewSettings: + m.view = viewDocs + default: + m.view = viewOverview + } + m.contentScroll = 0 + m.clampNetworkNav() + return m, nil + case "alt+up": + if m.contentScroll > 0 { + m.contentScroll-- + } + return m, nil + case "alt+down": + m.contentScroll++ + return m, nil + case "alt+shift+up", "alt+pgup": + m.contentScroll -= 10 + if m.contentScroll < 0 { + m.contentScroll = 0 + } + return m, nil + case "alt+shift+down", "alt+pgdown": + m.contentScroll += 10 + return m, nil + case "alt+home": + m.contentScroll = 0 + return m, nil + case "r": + if m.loading { + m.setStatus("refresh already running") + return m, nil + } + m.loading = true + return m, m.fetchStatsCmd() + } + + if m.view == viewUsage { + return m.handleUsageKey(v) + } + + if m.view == viewLive { + return m.handleLiveKey(v) + } + + if m.view == viewSettings { + return m.handleSettingsKey(v) + } + + if m.view == viewDocs { + return m.handleDocsKey(v) + } + + if m.view == viewClusters { + switch v.String() { + case "o": + m.view = viewOverview + case "n": + m.view = viewNetwork + case "s": + m.view = viewSettings + case "d", "D": + m.view = viewDocs + } + return m, nil + } + + if m.view == viewOverview { + switch v.String() { + case "left", "h", "p": + if m.hasStats { + m.runtime.selectedIface = cycleIface(interfaceNames(m.stats.Network), m.runtime.selectedIface, -1) + } + case "right", "l", "n": + if m.hasStats { + m.runtime.selectedIface = cycleIface(interfaceNames(m.stats.Network), m.runtime.selectedIface, 1) + } + case "N": + m.view = viewNetwork + } + return m, nil + } + + switch v.String() { + case "o": + m.view = viewOverview + case "d": + m.view = viewDocs + case "up", "k": + m.pinnedIface = "" + m.moveCursor(-1) + case "down", "j": + m.pinnedIface = "" + m.moveCursor(1) + case "left", "h": + m.metric = prevMetric(m.metric) + case "right", "l": + m.metric = nextMetric(m.metric) + case "p": + m.pinnedIface = "" + m.changePage(-1) + case "n": + m.pinnedIface = "" + m.changePage(1) + case "pgup": + m.pinnedIface = "" + m.changePage(-1) + case "pgdown": + m.pinnedIface = "" + m.changePage(1) + case "1": + m.metric = metricRX + case "2": + m.metric = metricTX + case "3": + m.metric = metricTotal + case "enter": + m.toggleIfacePin() + } + return m, nil +} + +func (m monitorModel) handleTerminalKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "ctrl+t": + if m.termFull { + m.termMode = false + m.termFull = false + m.setStatus("console hidden") + return m, nil + } + m.termFull = true + m.setStatus("console fullscreen") + return m, nil + case "ctrl+g": + m.termMode = false + m.termFull = false + m.setStatus("back to UI") + return m, nil + case "esc": + m.termMode = false + m.termFull = false + m.setStatus("console hidden") + return m, nil + case "ctrl+c": + m.termPartial = "" + m.termCursor = 0 + m.termHistPos = -1 + m.setStatus("input cleared") + return m, nil + case "alt+p": + m.privacyMode = !m.privacyMode + if m.privacyMode { + m.setStatus("privacy: on") + } else { + m.setStatus("privacy: off") + } + return m, nil + } + + switch v.String() { + case "alt+up": + m.scrollConsole(1) + return m, nil + case "alt+down": + m.scrollConsole(-1) + return m, nil + case "alt+shift+up": + m.scrollConsole(10) + return m, nil + case "alt+shift+down": + m.scrollConsole(-10) + return m, nil + case "pgup": + m.scrollConsole(m.dockOutputHeight() - 1) + return m, nil + case "pgdown": + m.scrollConsole(-(m.dockOutputHeight() - 1)) + return m, nil + case "alt+home": + m.termScroll = len(m.termLines) + m.clampTermScroll() + return m, nil + case "alt+end": + m.termScroll = 0 + return m, nil + } + + if m.termBusy { + return m, nil + } + + switch v.String() { + case "up": + m.historyUp() + return m, nil + case "down": + m.historyDown() + return m, nil + case "left": + m.moveConsoleCursor(-1) + return m, nil + case "right": + m.moveConsoleCursor(1) + return m, nil + case "alt+left", "alt+b": + m.moveConsoleWord(-1) + return m, nil + case "alt+right", "alt+f": + m.moveConsoleWord(1) + return m, nil + case "home", "ctrl+a": + m.termCursor = 0 + return m, nil + case "end", "ctrl+e": + m.termCursor = len([]rune(m.termPartial)) + return m, nil + case "backspace": + m.deleteConsolePrev() + return m, nil + case "delete": + m.deleteConsoleAt() + return m, nil + case "ctrl+u": + m.deleteConsoleToStart() + return m, nil + case "ctrl+k": + m.deleteConsoleToEnd() + return m, nil + case "ctrl+w": + m.deleteConsoleWord() + return m, nil + case "tab": + m.consoleAutocomplete() + return m, nil + case "enter": + return m.submitConsoleLine() + } + + switch v.Type { + case tea.KeyRunes: + if len(v.Runes) == 0 { + return m, nil + } + m.insertConsoleText(string(v.Runes)) + case tea.KeySpace: + m.insertConsoleText(" ") + } + return m, nil +} + +func (m monitorModel) ensureTerminalCmd() tea.Cmd { + if m.term != nil && m.termReady { + return nil + } + return m.startTerminalCmd() +} + +func (m monitorModel) startTerminalCmd() tea.Cmd { + cols := max(40, m.width-4) + rows := max(8, m.terminalRows()-2) + return func() tea.Msg { + shell := strings.TrimSpace(os.Getenv("SHELL")) + if shell == "" { + shell = "/bin/bash" + } + cmd := exec.Command(shell, "-i") + ptmx, err := pty.StartWithSize(cmd, &pty.Winsize{ + Cols: uint16(cols), + Rows: uint16(rows), + }) + if err != nil { + return terminalStartedMsg{err: err} + } + return terminalStartedMsg{ + session: &terminalSession{ + cmd: cmd, + ptmx: ptmx, + shell: shell, + }, + } + } +} + +func (m monitorModel) readTerminalCmd() tea.Cmd { + session := m.term + return func() tea.Msg { + if session == nil || session.ptmx == nil { + return terminalClosedMsg{} + } + buf := make([]byte, 4096) + n, err := session.ptmx.Read(buf) + if n > 0 { + return terminalOutputMsg{data: string(buf[:n])} + } + if err != nil { + if waitErr := session.cmd.Wait(); waitErr != nil && !errors.Is(waitErr, os.ErrClosed) { + return terminalClosedMsg{err: waitErr} + } + return terminalClosedMsg{} + } + return terminalOutputMsg{} + } +} + +func (m *monitorModel) appendTerminalText(chunk string) { + if chunk == "" { + return + } + + clean := sanitizeTerminalChunk(chunk) + if clean == "" { + return + } + + for _, r := range clean { + switch r { + case '\r': + m.termPartial = "" + case '\n': + m.pushTerminalLine(m.termPartial) + m.termPartial = "" + case '\b': + if len(m.termPartial) > 0 { + m.termPartial = m.termPartial[:len(m.termPartial)-1] + } + case '\t': + m.termPartial += " " + default: + m.termPartial += string(r) + } + } +} + +func (m *monitorModel) pushTerminalLine(line string) { + m.termLines = append(m.termLines, line) + const keep = 800 + if len(m.termLines) > keep { + m.termLines = m.termLines[len(m.termLines)-keep:] + } + m.termScroll = 0 +} + +func (m monitorModel) terminalRows() int { + if m.termFull { + return max(10, m.height-3) + } + return max(10, minInt(16, m.height/3)) +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func sanitizeTerminalChunk(chunk string) string { + if chunk == "" { + return "" + } + + s := ansiOSCRegex.ReplaceAllString(chunk, "") + s = ansiCSIRegex.ReplaceAllStringFunc(s, func(seq string) string { + if len(seq) > 0 && seq[len(seq)-1] == 'm' { + return seq + } + return "" + }) + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\x00", "") + return s +} + +func keyMsgToPTY(k tea.KeyMsg) []byte { + switch k.Type { + case tea.KeyRunes: + payload := []byte(string(k.Runes)) + if k.Alt { + return append([]byte{0x1b}, payload...) + } + return payload + case tea.KeySpace: + return []byte(" ") + case tea.KeyEnter: + return []byte{'\r'} + case tea.KeyTab: + return []byte{'\t'} + case tea.KeyShiftTab: + return []byte("\x1b[Z") + case tea.KeyBackspace: + return []byte{0x7f} + case tea.KeyUp: + return []byte("\x1b[A") + case tea.KeyDown: + return []byte("\x1b[B") + case tea.KeyRight: + return []byte("\x1b[C") + case tea.KeyLeft: + return []byte("\x1b[D") + case tea.KeyHome: + return []byte("\x1b[H") + case tea.KeyEnd: + return []byte("\x1b[F") + case tea.KeyDelete: + return []byte("\x1b[3~") + case tea.KeyInsert: + return []byte("\x1b[2~") + case tea.KeyPgUp: + return []byte("\x1b[5~") + case tea.KeyPgDown: + return []byte("\x1b[6~") + case tea.KeyCtrlC: + return []byte{0x03} + case tea.KeyCtrlD: + return []byte{0x04} + case tea.KeyCtrlF: + return []byte{0x06} + case tea.KeyCtrlL: + return []byte{0x0c} + case tea.KeyCtrlK: + return []byte{0x0b} + case tea.KeyCtrlN: + return []byte{0x0e} + case tea.KeyCtrlP: + return []byte{0x10} + case tea.KeyCtrlR: + return []byte{0x12} + case tea.KeyCtrlS: + return []byte{0x13} + case tea.KeyCtrlU: + return []byte{0x15} + case tea.KeyCtrlV: + return []byte{0x16} + case tea.KeyCtrlW: + return []byte{0x17} + case tea.KeyCtrlX: + return []byte{0x18} + case tea.KeyCtrlY: + return []byte{0x19} + case tea.KeyCtrlZ: + return []byte{0x1a} + case tea.KeyCtrlA: + return []byte{0x01} + case tea.KeyCtrlB: + return []byte{0x02} + case tea.KeyCtrlE: + return []byte{0x05} + case tea.KeyCtrlQ: + return []byte{0x11} + default: + if k.Type >= tea.KeyCtrlA && k.Type <= tea.KeyCtrlZ { + return []byte{byte(k.Type)} + } + if k.String() == "esc" { + return []byte{0x1b} + } + } + return nil +} + +func (s *terminalSession) close() { + if s == nil { + return + } + if s.ptmx != nil { + _ = s.ptmx.Close() + } + if s.cmd != nil && s.cmd.Process != nil { + _ = s.cmd.Process.Kill() + _, _ = s.cmd.Process.Wait() + } +} + +func (m monitorModel) handleInputKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "esc": + m.lockerPending = "" + m.inputMode = inputNone + m.inputBuf = "" + return m, nil + case "enter": + value := strings.TrimSpace(m.inputBuf) + mode := m.inputMode + m.inputMode = inputNone + m.inputBuf = "" + if mode == inputSearch { + m.searchTerm = value + m.page = 1 + m.cursor = 0 + m.clampNetworkNav() + m.setStatus(fmt.Sprintf("search=%q", m.searchTerm)) + return m, nil + } + if mode == inputLockerNew { + if len(value) < 4 { + m.inputMode = inputLockerNew + m.setStatus("locker password too short (min 4)") + return m, nil + } + m.lockerPending = value + m.inputMode = inputLockerConfirm + m.setStatus("locker: confirm password and press Enter") + return m, nil + } + if mode == inputLockerConfirm { + if value == "" || m.lockerPending == "" { + m.setStatus("locker setup cancelled") + return m, nil + } + if value != m.lockerPending { + m.lockerPending = "" + m.setStatus("locker passwords do not match") + return m, nil + } + m.lockerPending = "" + cfg, err := m.svc.SetLockerPassword(value) + if err != nil { + m.setStatus("locker setup failed: " + err.Error()) + return m, nil + } + m.lockerHash = strings.TrimSpace(cfg.PasswordHash) + m.lockerEnabled = cfg.Enabled + if err := m.saveTUISettings(); err != nil { + m.setStatus("save settings failed: " + err.Error()) + return m, nil + } + m.setStatus("locker password set and enabled") + return m, nil + } + return m, nil + case "backspace": + if len(m.inputBuf) > 0 { + _, size := utf8.DecodeLastRuneInString(m.inputBuf) + if size > 0 { + m.inputBuf = m.inputBuf[:len(m.inputBuf)-size] + } + } + return m, nil + } + + if v.Type == tea.KeyRunes && len(v.Runes) > 0 { + m.inputBuf += string(v.Runes) + return m, nil + } + return m, nil +} + +func (m monitorModel) handleLockKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "q", "ctrl+c": + return m, tea.Quit + case "backspace": + if len(m.lockInput) > 0 { + _, size := utf8.DecodeLastRuneInString(m.lockInput) + if size > 0 { + m.lockInput = m.lockInput[:len(m.lockInput)-size] + } + } + return m, nil + case "enter": + if strings.TrimSpace(m.lockerHash) == "" { + m.locked = false + m.lockErr = "" + return m, nil + } + if err := m.svc.UnlockLocker(m.lockInput); err != nil { + m.lockInput = "" + m.lockErr = "invalid password" + return m, nil + } + m.lockInput = "" + m.lockErr = "" + m.locked = false + m.lastUnlockAt = time.Now() + m.setStatus("unlocked") + return m, nil + } + if v.Type == tea.KeyRunes && len(v.Runes) > 0 { + m.lockInput += string(v.Runes) + return m, nil + } + return m, nil +} + +func (m monitorModel) renderLockedView(width int) string { + if width < 68 { + width = 68 + } + mask := strings.Repeat("*", len([]rune(m.lockInput))) + lines := []string{ + "Session locked.", + "", + "Enter locker password and press Enter.", + "Press Ctrl+C to quit.", + "", + "Password: " + mask, + } + if strings.TrimSpace(m.lockErr) != "" { + lines = append(lines, "Error: "+m.lockErr) + } + return renderPanel("Locker", strings.Join(lines, "\n"), width) +} + +func (m monitorModel) handleSettingsKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + const settingsRows = 6 + + switch v.String() { + case "up", "k": + m.settingsCursor-- + if m.settingsCursor < 0 { + m.settingsCursor = settingsRows - 1 + } + return m, nil + case "down", "j": + m.settingsCursor++ + if m.settingsCursor >= settingsRows { + m.settingsCursor = 0 + } + return m, nil + case "left", "h", "-", "_": + return m.adjustSetting(-1) + case "right", "l", "+", "=": + return m.adjustSetting(1) + case "enter": + if m.settingsCursor == 5 { + m.inputMode = inputLockerNew + m.inputBuf = "" + m.lockerPending = "" + m.setStatus("locker: enter new password and press Enter") + return m, nil + } + return m, nil + case "o": + m.view = viewOverview + return m, nil + case "c": + m.view = viewClusters + return m, nil + case "n": + m.view = viewNetwork + return m, nil + case "d", "D": + m.view = viewDocs + return m, nil + } + return m, nil +} + +func (m monitorModel) handleDocsKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + total := len(docsRawLines()) + viewport := m.docsViewportHeight() + switch v.String() { + case "up", "k": + m.docsScroll-- + case "down", "j": + m.docsScroll++ + case "pgup": + m.docsScroll -= max(1, viewport-2) + case "pgdown", "space": + m.docsScroll += max(1, viewport-2) + case "home", "g": + m.docsScroll = 0 + case "end", "G": + m.docsScroll = total + case "o": + m.view = viewOverview + return m, nil + case "c": + m.view = viewClusters + return m, nil + case "n": + m.view = viewNetwork + return m, nil + case "s": + m.view = viewSettings + return m, nil + } + maxOffset := total - viewport + if maxOffset < 0 { + maxOffset = 0 + } + if m.docsScroll < 0 { + m.docsScroll = 0 + } + if m.docsScroll > maxOffset { + m.docsScroll = maxOffset + } + return m, nil +} + +func (m monitorModel) adjustSetting(delta int) (tea.Model, tea.Cmd) { + if delta == 0 { + return m, nil + } + + switch m.settingsCursor { + case 0: + steps := []time.Duration{ + 500 * time.Millisecond, + time.Second, + 2 * time.Second, + 5 * time.Second, + 10 * time.Second, + 20 * time.Second, + 30 * time.Second, + 60 * time.Second, + 120 * time.Second, + } + idx := closestDurationIndex(steps, m.interval) + idx += delta + if idx < 0 { + idx = 0 + } + if idx >= len(steps) { + idx = len(steps) - 1 + } + m.interval = steps[idx] + m.setStatus("refresh interval: " + m.interval.String()) + case 1: + m.pageSize += delta + if m.pageSize < 5 { + m.pageSize = 5 + } + if m.pageSize > 50 { + m.pageSize = 50 + } + m.clampNetworkNav() + m.setStatus(fmt.Sprintf("network page size: %d", m.pageSize)) + case 2: + labels := []string{"1h", "24h", "7d", "30d", "all"} + cur := 0 + for i, label := range labels { + if strings.EqualFold(m.historyRange.Label, label) { + cur = i + break + } + } + cur += delta + if cur < 0 { + cur = 0 + } + if cur >= len(labels) { + cur = len(labels) - 1 + } + r, err := parseHistoryRange(labels[cur]) + if err == nil { + m.historyRange = r + m.page = 1 + m.cursor = 0 + m.setStatus("history range: " + r.Label) + _ = m.saveTUISettings() + return m, m.loadHistoryCmd() + } + case 3: + if m.telegramSyncing { + m.setStatus("telegram update in progress") + return m, nil + } + cfg := m.telegram + if strings.TrimSpace(cfg.Token) == "" || len(cfg.AllowedUserIDs) == 0 { + m.setStatus("telegram is not configured; use `bot telegram set --token ... --allow ...`") + return m, nil + } + targetEnabled := cfg.Enabled + if delta > 0 { + targetEnabled = true + } else if delta < 0 { + targetEnabled = false + } + if targetEnabled == cfg.Enabled { + return m, nil + } + m.telegramSyncing = true + m.setStatus("telegram bot: applying setting...") + return m, m.syncTelegramSettingCmd(targetEnabled) + case 4: + if strings.TrimSpace(m.lockerHash) == "" { + m.setStatus("set locker password first (row 'Locker password')") + return m, nil + } + targetEnabled := m.lockerEnabled + if delta > 0 { + targetEnabled = true + } else if delta < 0 { + targetEnabled = false + } + if targetEnabled == m.lockerEnabled { + return m, nil + } + cfg, err := m.svc.SetLockerEnabled(targetEnabled) + if err != nil { + m.setStatus("locker update failed: " + err.Error()) + return m, nil + } + m.lockerEnabled = cfg.Enabled + m.lockerHash = strings.TrimSpace(cfg.PasswordHash) + if !m.lockerEnabled { + m.locked = false + m.lockInput = "" + m.lockErr = "" + _ = m.svc.LockNow() + } else { + m.lastUnlockAt = time.Now() + } + m.setStatus("tui locker: " + ternary(m.lockerEnabled, "enabled", "disabled")) + case 5: + m.setStatus("press Enter to set locker password") + } + + if err := m.saveTUISettings(); err != nil { + m.setStatus("save settings failed: " + err.Error()) + } + return m, nil +} + +func (m monitorModel) syncTelegramSettingCmd(targetEnabled bool) tea.Cmd { + cfg := m.telegram + return func() tea.Msg { + cfg.Enabled = targetEnabled + updated, err := m.svc.SetTelegram(cfg) + if err != nil { + return telegramSyncMsg{err: err} + } + if err := syncTelegramBotDaemon(m.svc, m.svc.ConfigPath(), updated.Enabled, telegramBotDefaultPoll); err != nil { + return telegramSyncMsg{updated: updated, err: err} + } + return telegramSyncMsg{updated: updated} + } +} + +func closestDurationIndex(items []time.Duration, current time.Duration) int { + if len(items) == 0 { + return 0 + } + bestIdx := 0 + bestDist := absDuration(items[0] - current) + for i := 1; i < len(items); i++ { + d := absDuration(items[i] - current) + if d < bestDist { + bestDist = d + bestIdx = i + } + } + return bestIdx +} + +func absDuration(v time.Duration) time.Duration { + if v < 0 { + return -v + } + return v +} + +func (m monitorModel) saveTUISettings() error { + if strings.TrimSpace(m.settingsPath) == "" { + return nil + } + payload := tuiSettings{ + RefreshMillis: int(m.interval.Milliseconds()), + PageSize: m.pageSize, + HistoryRange: m.historyRange.Label, + } + return saveTUISettings(m.settingsPath, payload) +} + +func loadTUISettings(path string) (tuiSettings, error) { + if strings.TrimSpace(path) == "" { + return tuiSettings{}, errors.New("settings path is empty") + } + raw, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return tuiSettings{}, nil + } + return tuiSettings{}, err + } + var cfg tuiSettings + if err := json.Unmarshal(raw, &cfg); err != nil { + return tuiSettings{}, err + } + return cfg, nil +} + +func saveTUISettings(path string, cfg tuiSettings) error { + if strings.TrimSpace(path) == "" { + return errors.New("settings path is empty") + } + if cfg.RefreshMillis < 500 { + cfg.RefreshMillis = 500 + } + if cfg.PageSize < 5 { + cfg.PageSize = 5 + } + if cfg.PageSize > 50 { + cfg.PageSize = 50 + } + + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + body, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return err + } + body = append(body, '\n') + return os.WriteFile(path, body, 0o600) +} + +func (m *monitorModel) setStatus(msg string) { + m.statusMsg = strings.TrimSpace(msg) +} + +func (m *monitorModel) moveCursor(delta int) { + rows := m.networkRows() + if len(rows) == 0 { + m.cursor = 0 + return + } + + start, end, _ := paginationBounds(len(rows), m.page, m.pageSize) + pageLen := end - start + if pageLen <= 0 { + m.cursor = 0 + return + } + + m.cursor += delta + if m.cursor < 0 { + if m.page > 1 { + m.page-- + start, end, _ = paginationBounds(len(rows), m.page, m.pageSize) + m.cursor = max(0, (end-start)-1) + } else { + m.cursor = 0 + } + return + } + if m.cursor >= pageLen { + if m.page < pageCount(len(rows), m.pageSize) { + m.page++ + m.cursor = 0 + } else { + m.cursor = pageLen - 1 + } + } +} + +func (m *monitorModel) changePage(delta int) { + rows := m.networkRows() + totalPages := pageCount(len(rows), m.pageSize) + if totalPages < 1 { + totalPages = 1 + } + m.page += delta + if m.page < 1 { + m.page = 1 + } + if m.page > totalPages { + m.page = totalPages + } + m.cursor = 0 +} + +// toggleIfacePin pins the cursor to the currently highlighted network +// interface so auto-refresh re-sorting can't move the selection onto a +// different one. Pressing Enter again clears the pin. +func (m *monitorModel) toggleIfacePin() { + rows := m.networkRows() + if len(rows) == 0 { + m.pinnedIface = "" + m.setStatus("no interface to pin") + return + } + start, end, _ := paginationBounds(len(rows), m.page, m.pageSize) + pageRows := rows[start:end] + if m.cursor < 0 || m.cursor >= len(pageRows) { + m.pinnedIface = "" + m.setStatus("no interface to pin") + return + } + current := pageRows[m.cursor].Interface + if m.pinnedIface == current { + m.pinnedIface = "" + m.setStatus("iface unlocked: " + current) + return + } + m.pinnedIface = current + m.setStatus("iface locked: " + current + " (Enter to unlock)") +} + +// snapToPinnedIface repositions page/cursor so the pinned interface is the +// highlighted row after sort order changes. No-op if nothing is pinned or the +// pinned interface disappeared from the current result set. +func (m *monitorModel) snapToPinnedIface() { + if strings.TrimSpace(m.pinnedIface) == "" { + return + } + rows := m.networkRows() + if len(rows) == 0 { + return + } + idx := -1 + for i, r := range rows { + if r.Interface == m.pinnedIface { + idx = i + break + } + } + if idx < 0 { + // Pinned interface disappeared (filtered out or went away). + m.pinnedIface = "" + return + } + if m.pageSize <= 0 { + return + } + m.page = (idx / m.pageSize) + 1 + m.cursor = idx % m.pageSize +} + +func (m *monitorModel) clampNetworkNav() { + rows := m.networkRows() + totalPages := pageCount(len(rows), m.pageSize) + if totalPages < 1 { + totalPages = 1 + } + if m.page < 1 { + m.page = 1 + } + if m.page > totalPages { + m.page = totalPages + } + + start, end, _ := paginationBounds(len(rows), m.page, m.pageSize) + pageLen := end - start + if pageLen <= 0 { + m.cursor = 0 + return + } + if m.cursor < 0 { + m.cursor = 0 + } + if m.cursor >= pageLen { + m.cursor = pageLen - 1 + } +} + +func (m monitorModel) fetchStatsCmd() tea.Cmd { + svc := m.svc + name := m.cluster.Name + return func() tea.Msg { + stats, err := fetchAgentStats(svc, name) + return monitorStatsMsg{stats: stats, err: err} + } +} + +func (m monitorModel) fetchVMAlertCmd() tea.Cmd { + svc := m.svc + clusterID := m.cluster.ID + return func() tea.Msg { + if strings.TrimSpace(clusterID) == "" { + return vmAlertMsg{} + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + rep, err := svc.CheckVMAlerts(ctx, clusterID) + if err != nil { + return vmAlertMsg{err: err} + } + return vmAlertMsg{warnings: rep.Warnings} + } +} + +func (m monitorModel) nextTickCmd() tea.Cmd { + return tea.Tick(m.interval, func(time.Time) tea.Msg { + return monitorTickMsg{} + }) +} + +func spinnerTickCmd() tea.Cmd { + return tea.Tick(200*time.Millisecond, func(time.Time) tea.Msg { + return spinnerTickMsg{} + }) +} + +var spinnerSymbols = []string{"✻", "✦", "✧", "✦"} + +var thinkingPhrases = []string{ + "Thinking", + "Pondering", + "Cogitating", + "Musing", + "Brewing", + "Contemplating", + "Ruminating", + "Synthesizing", + "Deliberating", +} + +func gradientText(text string, frame int) string { + runes := []rune(text) + if len(runes) == 0 { + return "" + } + const ( + r1, g1, b1 = 217, 119, 87 + r2, g2, b2 = 255, 220, 150 + ) + var b strings.Builder + for i, r := range runes { + phase := float64(i)*0.55 - float64(frame)*0.35 + t := (math.Cos(phase) + 1) / 2 + rr := int(float64(r1)*(1-t) + float64(r2)*t) + gg := int(float64(g1)*(1-t) + float64(g2)*t) + bb := int(float64(b1)*(1-t) + float64(b2)*t) + hex := fmt.Sprintf("#%02X%02X%02X", rr, gg, bb) + b.WriteString(lipgloss.NewStyle().Foreground(lipgloss.Color(hex)).Bold(true).Render(string(r))) + } + return b.String() +} + +func renderThinkingSpinner(frame int, width int, elapsed int) string { + sym := spinnerSymbols[(frame/2)%len(spinnerSymbols)] + phrase := thinkingPhrases[(frame/15)%len(thinkingPhrases)] + dots := strings.Repeat(".", (frame/3)%4) + text := phrase + dots + meta := softStyle.Render(fmt.Sprintf("(%2ds · esc to interrupt)", elapsed)) + line := accentStyle.Bold(true).Render(sym) + " " + gradientText(text, frame/2) + " " + meta + if width <= 0 { + return line + } + if visibleLen(line) > width { + line = truncateVisible(line, width) + } + if pad := width - visibleLen(line); pad > 0 { + line += strings.Repeat(" ", pad) + } + return line +} + +func (m monitorModel) appendHistoryCmd(snap history.NetworkSnapshot) tea.Cmd { + store := m.historyStore + clusterID := m.cluster.ID + return func() tea.Msg { + if store == nil { + return historyAppendMsg{} + } + return historyAppendMsg{err: store.Append(clusterID, snap)} + } +} + +func (m monitorModel) loadHistoryCmd() tea.Cmd { + store := m.historyStore + clusterID := m.cluster.ID + since := m.historyRange.Since + if m.historyRange.All { + since = time.Time{} + } + return func() tea.Msg { + if store == nil { + return historyLoadedMsg{points: map[string][]ifaceHistoryPoint{}, count: 0} + } + snaps, err := store.Load(clusterID, since) + if err != nil { + return historyLoadedMsg{err: err} + } + return historyLoadedMsg{ + points: buildHistoryPoints(snaps), + count: len(snaps), + } + } +} + +func buildHistoryPoints(snaps []history.NetworkSnapshot) map[string][]ifaceHistoryPoint { + points := make(map[string][]ifaceHistoryPoint, 64) + for _, snap := range snaps { + for _, iface := range snap.Interfaces { + points[iface.Interface] = append(points[iface.Interface], ifaceHistoryPoint{ + At: snap.Timestamp, + RxMbps: iface.RxMbps, + TxMbps: iface.TxMbps, + RxDrops: iface.RxDrops, + TxDrops: iface.TxDrops, + }) + } + } + return points +} + +func (m *monitorModel) makeHistorySnapshot(stats agent.StatsResponse) history.NetworkSnapshot { + ts := stats.Timestamp + if ts.IsZero() { + ts = time.Now().UTC() + } + + byIface := make(map[string]agent.NetworkStat, len(stats.Network)) + for _, n := range stats.Network { + byIface[n.Interface] = n + } + + items := make([]history.InterfaceSample, 0, len(m.runtime.rates)) + names := make([]string, 0, len(m.runtime.rates)) + for iface := range m.runtime.rates { + names = append(names, iface) + } + sort.Strings(names) + for _, iface := range names { + rate := m.runtime.rates[iface] + n := byIface[iface] + items = append(items, history.InterfaceSample{ + Interface: iface, + RxMbps: rate.RxMbps, + TxMbps: rate.TxMbps, + RxDrops: n.RxDrops, + TxDrops: n.TxDrops, + }) + } + + return history.NetworkSnapshot{ + Timestamp: ts, + Interfaces: items, + } +} + +func (m *monitorModel) applyHistorySnapshot(snap history.NetworkSnapshot) { + if len(snap.Interfaces) == 0 { + return + } + if m.history == nil { + m.history = map[string][]ifaceHistoryPoint{} + } + + for _, iface := range snap.Interfaces { + m.history[iface.Interface] = append(m.history[iface.Interface], ifaceHistoryPoint{ + At: snap.Timestamp, + RxMbps: iface.RxMbps, + TxMbps: iface.TxMbps, + RxDrops: iface.RxDrops, + TxDrops: iface.TxDrops, + }) + } +} + +type dashboardSection struct { + Title string + Body string +} + +func (m monitorModel) viewLabel() string { + switch m.view { + case viewClusters: + return "clusters" + case viewNetwork: + return "network" + case viewSettings: + return "settings" + case viewDocs: + return "docs" + case viewUsage: + return "usage" + case viewLive: + return "live" + default: + return "overview" + } +} + +func splitMainGridWidths(width int) (leftW, rightW int) { + if width < 70 { + return width, 0 + } + + leftW = int(math.Round(float64(width) * 0.70)) + rightW = width - leftW - 1 + + if rightW < 24 { + rightW = 24 + leftW = width - rightW - 1 + } + if leftW < 40 { + leftW = 40 + rightW = width - leftW - 1 + } + if rightW < 0 { + rightW = 0 + } + return leftW, rightW +} + +func renderSectionsBody(width int, sections []dashboardSection) string { + contentWidth := max(18, width-8) + sep := dimStyle.Render(strings.Repeat("─", max(8, contentWidth))) + + lines := make([]string, 0, len(sections)*3) + for i, s := range sections { + if i > 0 { + lines = append(lines, sep) + } + lines = append(lines, titleStyle.Render(s.Title)) + body := strings.TrimSpace(s.Body) + if body == "" { + body = dimStyle.Render("n/a") + } + lines = append(lines, truncateMultiline(body, contentWidth)) + } + return lipgloss.JoinVertical(lipgloss.Left, lines...) +} + +func (m monitorModel) headerBody(width int) string { + target := fmt.Sprintf("%s@%s:%d", m.cluster.User, m.cluster.Host, m.cluster.Port) + if !m.hasCluster { + target = "(not connected)" + } + ts := "-" + if m.hasStats && !m.stats.Timestamp.IsZero() { + ts = m.stats.Timestamp.Local().Format("2006-01-02 15:04:05") + } + + status := okStyle.Render("OK") + if m.loading { + status = warnStyle.Render("UPDATING") + } + if m.lastErr != nil { + status = critStyle.Render("ERROR") + } + + mode := m.viewLabel() + + body := strings.Join([]string{ + fmt.Sprintf("✻ Welcome to PXmon (Phylex Monitor)"), + fmt.Sprintf("cluster: %s (%s)", m.cluster.Name, target), + fmt.Sprintf("mode: %s · refresh: %s · range: %s · status: %s · time: %s", mode, m.interval, m.historyRange.Label, status, ts), + }, "\n") + if m.lastErr != nil { + errLine := "Last error: " + m.lastErr.Error() + body += "\n" + critStyle.Render(truncate(errLine, max(24, width-6))) + } + return truncateMultiline(body, max(24, width-8)) +} + +func (m monitorModel) renderHeader(width int) string { + const headerHeight = 8 + panel, _ := renderFixedPanel(headerPanelStyle, "Session Header", m.headerBody(width), width, headerHeight, 0) + return panel +} + +func (m monitorModel) mainLeftPanel(width int) (string, string) { + if !m.hasCluster { + return "Workspace", renderSectionsBody(width, []dashboardSection{ + { + Title: "Welcome", + Body: m.welcomeBody(width), + }, + { + Title: "Connect Cluster", + Body: "No active cluster.\n\nOpen command console (`t`) and run:\ncluster connect --name eu-1 --host --user root --auth key --key-path ~/.ssh/id_ed25519", + }, + }) + } + + switch m.view { + case viewClusters: + return "Cluster Overview", m.clusterMainBody(width) + case viewNetwork: + return "Network View", m.networkMainBody(width) + case viewSettings: + return "Settings", m.settingsMainBody(width) + case viewDocs: + return "Docs", strings.TrimSpace(strings.Join(docsRawLines(), "\n")) + default: + return "Overview", m.overviewMainBody(width) + } +} + +func (m monitorModel) mainRightPanel(width int) string { + clusterLine := m.cluster.Name + if !m.hasCluster { + clusterLine = "(none)" + } + + statusLine := m.statusMsg + if strings.TrimSpace(statusLine) == "" { + statusLine = "ready" + } + + activity := strings.Join([]string{ + fmt.Sprintf("cluster: %s", clusterLine), + fmt.Sprintf("mode: %s", m.viewLabel()), + fmt.Sprintf("alerts: %d", len(collectAlerts(m.stats, &m.runtime))), + fmt.Sprintf("history: %d cmds", len(m.termHistory)), + fmt.Sprintf("telegram: %s", m.telegramStatusSummary()), + fmt.Sprintf("status: %s", truncate(statusLine, max(10, width-16))), + }, "\n") + + hotkeys := []string{ + "tab cycle views", + "t focus console", + "ctrl+t console fullscreen", + "ctrl+g leave console", + "c/s/d/o clusters/settings/docs/overview", + "r refresh stats", + "q quit", + } + if m.view == viewNetwork { + hotkeys = append(hotkeys, "/ search interfaces") + hotkeys = append(hotkeys, "n/p page next/prev") + hotkeys = append(hotkeys, "enter lock/unlock iface") + } + if m.view == viewSettings { + hotkeys = append(hotkeys, "↑/↓ select setting") + hotkeys = append(hotkeys, "←/→ change value") + } + + return renderSectionsBody(width, []dashboardSection{ + {Title: "Activity", Body: activity}, + {Title: "Hotkeys", Body: strings.Join(hotkeys, "\n")}, + }) +} + +func (m monitorModel) overviewMainBody(width int) string { + return renderSectionsBody(width, []dashboardSection{ + {Title: "Welcome", Body: m.welcomeBody(width)}, + {Title: "System Snapshot", Body: m.snapshotBody(width)}, + {Title: "Network Top-5", Body: m.overviewNetworkBody(width)}, + {Title: "Alerts", Body: m.alertsBody(width)}, + }) +} + +func (m monitorModel) clusterMainBody(width int) string { + clusters, activeID, err := m.svc.List() + if err != nil { + return renderSectionsBody(width, []dashboardSection{ + {Title: "Error", Body: "failed to load cluster inventory: " + err.Error()}, + }) + } + + total := len(clusters) + agents := 0 + softwareDetected := 0 + activeName := "(none)" + for _, c := range clusters { + if c.ID == activeID { + activeName = c.Name + } + if c.Agent.Installed { + agents++ + } + if c.Software.Summary() != "-" && c.Software.Summary() != "none" { + softwareDetected++ + } + } + + summary := strings.Join([]string{ + fmt.Sprintf("clusters total: %d", total), + fmt.Sprintf("active cluster: %s", activeName), + fmt.Sprintf("agent installed: %d/%d", agents, max(total, 1)), + fmt.Sprintf("software detected: %d/%d", softwareDetected, max(total, 1)), + fmt.Sprintf("telegram bot: %s", m.telegramStatusSummary()), + }, "\n") + + if total == 0 { + return renderSectionsBody(width, []dashboardSection{ + {Title: "Summary", Body: summary}, + { + Title: "Clusters", + Body: "No clusters connected yet.\nUse command console (`t`) and run:\ncluster connect --name --host --user root --auth key --key-path ~/.ssh/id_ed25519", + }, + }) + } + + var table strings.Builder + table.WriteString(fmt.Sprintf("%-2s %-14s %-24s %-9s %-18s %-11s\n", "A", "NAME", "TARGET", "AGENT", "SOFTWARE", "UPDATED")) + for _, c := range clusters { + active := " " + if c.ID == activeID { + active = "*" + } + agentState := "no" + if c.Agent.Installed { + agentState = fmt.Sprintf("yes:%d", c.Agent.Port) + } + target := truncate(fmt.Sprintf("%s@%s:%d", c.User, c.Host, c.Port), 24) + updated := c.UpdatedAt.Local().Format("01-02 15:04") + table.WriteString(fmt.Sprintf("%-2s %-14s %-24s %-9s %-18s %-11s\n", + active, + truncate(c.Name, 14), + target, + truncate(agentState, 9), + truncate(c.Software.Summary(), 18), + updated, + )) + } + + return renderSectionsBody(width, []dashboardSection{ + {Title: "Summary", Body: summary}, + {Title: "Clusters", Body: strings.TrimRight(table.String(), "\n")}, + }) +} + +func (m monitorModel) networkMainBody(width int) string { + rows := m.networkRows() + start, end, totalPages := paginationBounds(len(rows), m.page, m.pageSize) + pageRows := m.enrichNetworkRows(rows[start:end]) + + meta := []string{ + fmt.Sprintf("search=%q", m.searchTerm), + fmt.Sprintf("page=%d/%d", max(1, m.page), max(1, totalPages)), + fmt.Sprintf("interfaces=%d", len(rows)), + fmt.Sprintf("range=%s", m.historyRange.Label), + m.metricTabs(), + "search: press `/`, type query, Enter apply, Esc cancel", + } + + var table strings.Builder + table.WriteString(fmt.Sprintf("%-2s %-12s %8s %8s %8s %8s %8s %8s %-16s\n", + "", "iface", "rx", "tx", "total", "avg", "peak", "used", "graph")) + for i, row := range pageRows { + marker := " " + if i == m.cursor { + marker = ">" + } + table.WriteString(fmt.Sprintf("%-2s %-12s %8.2f %8.2f %8.2f %8.2f %8.2f %8s %-16s\n", + marker, + truncate(row.Interface, 12), + row.CurRxMbps, + row.CurTxMbps, + row.CurTotalMbps, + row.AvgTotalMbps, + row.PeakTotal, + humanBitsRate(row.ConsumedByte), + truncate(row.Spark, 16), + )) + } + if len(pageRows) == 0 { + table.WriteString(" " + dimStyle.Render("no interfaces matched current filter")) + } + + return renderSectionsBody(width, []dashboardSection{ + {Title: "Controls", Body: strings.Join(meta, "\n")}, + {Title: "Interfaces", Body: strings.TrimRight(table.String(), "\n")}, + {Title: "Details", Body: m.networkDetailsBody(pageRows, width)}, + }) +} + +func (m monitorModel) settingsMainBody(width int) string { + rows := []string{ + fmt.Sprintf("Refresh interval: %s", m.interval), + fmt.Sprintf("Network page size: %d", m.pageSize), + fmt.Sprintf("History range: %s", m.historyRange.Label), + fmt.Sprintf("Telegram bot: %s", m.telegramStatusSummary()), + fmt.Sprintf("TUI locker: %s", m.lockerStatusSummary()), + fmt.Sprintf("Locker password: %s", ternary(strings.TrimSpace(m.lockerHash) == "", "not set (press Enter)", "set (press Enter to replace)")), + } + + var body strings.Builder + body.WriteString("Use up/down to select setting, left/right (+/-) to change.\n") + body.WriteString("Settings are persisted locally.\n\n") + for i, row := range rows { + prefix := " " + if i == m.settingsCursor { + prefix = ">" + } + body.WriteString(fmt.Sprintf("%s %s\n", prefix, row)) + } + body.WriteString("\n") + body.WriteString("Configure telegram token and allowed users in console:\n") + body.WriteString("bot telegram set --token --allow --allow ") + + return renderSectionsBody(width, []dashboardSection{ + {Title: "TUI Settings", Body: strings.TrimRight(body.String(), "\n")}, + }) +} + +func (m monitorModel) renderOverviewDashboard(width int) string { + leftW, sideW := splitWidths(width) + panelH := m.mainViewportHeight(width) + + body := renderSectionsBody(leftW, []dashboardSection{ + {Title: "Welcome", Body: m.welcomeBody(leftW)}, + {Title: "System Snapshot", Body: m.snapshotBody(leftW)}, + {Title: "Network Focus", Body: m.overviewNetworkBody(leftW)}, + {Title: "Alerts", Body: m.alertsBody(leftW)}, + }) + left, _ := renderFixedPanel(panelStyle, "Overview", body, leftW, panelH, m.contentScroll) + + side := "" + if sideW > 0 { + side, _ = renderFixedPanel(panelStyle, "Tips & Activity", m.activityPanelBody(sideW), sideW, panelH, m.contentScroll) + } + return composeWithSidebar(left, side, width, leftW, sideW) +} + +func (m monitorModel) welcomeBody(width int) string { + lines := []string{ + "✻ Welcome to PXmon (Phylex Monitor)", + "", + "Tips for getting started:", + " • Press `t` to open the command console", + " • Run `cluster connect` to add a node over SSH", + " • Use `tab` to cycle overview / clusters / network / settings / docs", + " • Press `d` to open full Docs page", + " • Press `?` for shortcuts, `q` to quit", + } + out := strings.Join(lines, "\n") + return truncateMultiline(out, max(24, width-6)) +} + +func (m monitorModel) snapshotBody(width int) string { + if !m.hasStats { + return dimStyle.Render("waiting for first metrics sample...") + } + + cpu := m.currentCPUPct() + mem := m.stats.Memory + swap := m.stats.Memory.SwapUsedPct + if m.stats.Memory.SwapTotalBytes == 0 { + swap = 0 + } + diskUsed := 0.0 + diskLabel := "n/a" + if worst := worstDisk(m.stats.Disk); worst != nil { + diskUsed = worst.UsedPercent + diskLabel = truncate(worst.MountPoint, 14) + } + + barW := 18 + if width > 90 { + barW = 24 + } + + rows := []string{ + fmt.Sprintf("CPU %s %6.1f%%", barASCII(cpu, barW), cpu), + fmt.Sprintf("RAM %s %6.1f%%", barASCII(mem.UsedPercent, barW), mem.UsedPercent), + fmt.Sprintf("SWAP %s %6.1f%%", barASCII(swap, barW), swap), + fmt.Sprintf("DISK %s %6.1f%% (%s)", barASCII(diskUsed, barW), diskUsed, diskLabel), + } + + iface := strings.TrimSpace(m.runtime.selectedIface) + if iface != "" { + if rate, ok := m.runtime.rates[iface]; ok { + rows = append(rows, + "", + fmt.Sprintf("IFACE %s", iface), + fmt.Sprintf("RX %.2f Mbps | TX %.2f Mbps", rate.RxMbps, rate.TxMbps), + ) + } + } + return strings.Join(rows, "\n") +} + +func (m monitorModel) alertsBody(width int) string { + alerts := collectAlerts(m.stats, &m.runtime) + if len(alerts) == 0 { + return okStyle.Render("none") + } + + maxRows := 5 + if len(alerts) < maxRows { + maxRows = len(alerts) + } + var b strings.Builder + for i := 0; i < maxRows; i++ { + b.WriteString("• ") + b.WriteString(truncate(alerts[i], max(16, width-8))) + b.WriteByte('\n') + } + if len(alerts) > maxRows { + b.WriteString(dimStyle.Render(fmt.Sprintf("+%d more", len(alerts)-maxRows))) + } + return strings.TrimRight(b.String(), "\n") +} + +func (m monitorModel) activityPanelBody(width int) string { + alertCount := len(collectAlerts(m.stats, &m.runtime)) + statusLine := m.statusMsg + if strings.TrimSpace(statusLine) == "" { + statusLine = "ready" + } + + mode := "overview" + switch m.view { + case viewClusters: + mode = "clusters" + case viewNetwork: + mode = "network" + case viewSettings: + mode = "settings" + case viewDocs: + mode = "docs" + } + + last := "(none)" + if len(m.termLines) > 0 { + last = m.termLines[len(m.termLines)-1] + } + + lines := []string{ + "Activity", + fmt.Sprintf("mode: %s", mode), + fmt.Sprintf("alerts: %d", alertCount), + fmt.Sprintf("history: %d cmds", len(m.termHistory)), + fmt.Sprintf("telegram: %s", m.telegramStatusSummary()), + fmt.Sprintf("status: %s", truncate(statusLine, max(10, width-12))), + fmt.Sprintf("last: %s", truncate(last, max(10, width-10))), + "", + "Hotkeys", + "tab switch views", + "d docs page", + "t console focus", + "c clusters view", + "s settings view", + "/ search (network)", + "ctrl+g leave console", + } + + return strings.Join(lines, "\n") +} + +func (m monitorModel) renderCommandDock(width int) string { + state := dimStyle.Render("STANDBY") + title := "Command Console" + if m.termMode { + state = okStyle.Render("ACTIVE") + title = "Command Console (ACTIVE)" + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("state: %s · history: %d", state, len(m.termHistory))) + b.WriteByte('\n') + + if m.termMode { + outputHeight := m.dockOutputHeight() + innerWidth := width - 6 + if innerWidth < 20 { + innerWidth = 20 + } + spinner := "" + if m.termBusy { + elapsed := 0 + if !m.spinnerStart.IsZero() { + elapsed = int(time.Since(m.spinnerStart).Seconds()) + } + spinner = renderThinkingSpinner(m.thinkFrame, innerWidth, elapsed) + } + b.WriteString(m.renderDockOutput(outputHeight, innerWidth, spinner)) + b.WriteByte('\n') + } else if m.termBusy { + elapsed := 0 + if !m.spinnerStart.IsZero() { + elapsed = int(time.Since(m.spinnerStart).Seconds()) + } + b.WriteString(renderThinkingSpinner(m.thinkFrame, max(20, width-8), elapsed)) + b.WriteByte('\n') + } + + if m.termMode { + input := truncateVisible(m.renderConsoleInput(), max(20, width-6)) + b.WriteString(input) + } else { + b.WriteString(accentStyle.Render("> ")) + switch m.inputMode { + case inputSearch: + b.WriteString(dimStyle.Render("/" + m.inputBuf)) + case inputLockerNew, inputLockerConfirm: + b.WriteString(dimStyle.Render("locker password: " + strings.Repeat("*", len([]rune(m.inputBuf))))) + default: + b.WriteString(dimStyle.Render("press `t` to activate command console")) + } + } + + b.WriteByte('\n') + helpLine := m.helpText() + if strings.TrimSpace(m.statusMsg) != "" { + helpLine = "status: " + m.statusMsg + } + b.WriteString(dimStyle.Render(truncateRunes(helpLine, max(20, width-6)))) + + return renderPanelStyled(commandPanelStyle, title, b.String(), width) +} + +func (m monitorModel) dockOutputHeight() int { + if m.termFull { + h := m.height - 12 + if h < 10 { + h = 10 + } + return h + } + // Default 8 lines of output, but shrink when the terminal is small so + // that header + content + dock still fits without overflowing. + h := 8 + if m.height > 0 { + // Reserve ~11 lines for header (6) + content minimum (3) + dock chrome + // (5: title+state+input+status+border) + separators (2). That leaves + // m.height - 16 for dock output. + avail := m.height - 16 + if avail < h { + h = avail + } + if h < 3 { + h = 3 + } + } + return h +} + +func (m monitorModel) renderDockOutput(height, width int, spinner string) string { + if height <= 0 { + return "" + } + contentHeight := height + if spinner != "" && contentHeight > 1 { + contentHeight = height - 1 + } + + src := m.termLines + lines := make([]string, 0, height) + if len(src) == 0 { + lines = append(lines, dimStyle.Render("Observer console ready — type `help` or `cluster list`.")) + for len(lines) < contentHeight { + lines = append(lines, "") + } + } else { + total := len(src) + maxOffset := total - contentHeight + if maxOffset < 0 { + maxOffset = 0 + } + offset := m.termScroll + if offset < 0 { + offset = 0 + } + if offset > maxOffset { + offset = maxOffset + } + end := total - offset + start := end - contentHeight + if start < 0 { + start = 0 + } + window := src[start:end] + + for _, l := range window { + lines = append(lines, truncateVisible(l, width)) + } + for len(lines) < contentHeight { + lines = append(lines, "") + } + + if offset > 0 && contentHeight > 0 { + badge := softStyle.Render(fmt.Sprintf("↑ scrolled +%d (alt+↓ to follow)", offset)) + avail := width - visibleLen(badge) - 1 + if avail < 0 { + avail = 0 + } + lines[contentHeight-1] = strings.Repeat(" ", avail) + badge + } + } + + if spinner != "" { + lines = append(lines, truncateVisible(spinner, width)) + } + for len(lines) < height { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + +func splitWidths(width int) (leftW, sideW int) { + if width < 110 { + return width - 2, 0 + } + sideW = 30 + if width >= 150 { + sideW = 36 + } + leftW = width - sideW - 7 + if leftW < 68 { + return width - 2, 0 + } + return leftW, sideW +} + +func composeWithSidebar(left, side string, totalWidth, leftW, sideW int) string { + if sideW <= 0 || leftW <= 0 || strings.TrimSpace(side) == "" { + return left + } + return lipgloss.JoinHorizontal(lipgloss.Top, left, " ", side) +} + +func (m monitorModel) helpText() string { + const scroll = " | alt+↑/↓ scroll content" + help := "keys: t console | ctrl+t fullscreen console | tab view | d docs | c clusters | s settings | r refresh | q quit" + scroll + if m.view == viewOverview { + help = "keys: left/right iface | tab view | d docs | c clusters | s settings | t console | ctrl+t fullscreen | r refresh | q quit" + scroll + } else if m.view == viewClusters { + help = "keys: tab view | o overview | n network | s settings | d docs | t console | ctrl+t fullscreen | r refresh | q quit" + scroll + } else if m.view == viewNetwork { + help = "keys: up/down row | left/right or 1/2/3 | n/p page | / search | d docs | c clusters | s settings | t console | ctrl+t fullscreen | r refresh | q quit" + scroll + } else if m.view == viewSettings { + help = "keys: up/down setting | left/right adjust | Enter set locker password | d docs | tab view | c clusters | t console | ctrl+t fullscreen | q quit" + scroll + } else if m.view == viewDocs { + help = "keys: ↑/↓ or j/k scroll | PgUp/PgDn | Home/End | o/c/n/s switch view | tab cycle | t console | q quit" + } + if m.inputMode == inputSearch { + help = "search mode: type query | Enter apply | Esc cancel" + } + if m.inputMode == inputLockerNew { + help = "locker setup: type new password | Enter continue | Esc cancel" + } + if m.inputMode == inputLockerConfirm { + help = "locker setup: confirm password | Enter save | Esc cancel" + } + if m.termMode { + help = "console: commands | console full/dock/toggle | end line with \\ to continue | alt+←/→ word move | alt+↑/↓ scroll | ctrl+g back" + } + return truncate(help, max(20, m.width-16)) +} + +func (m monitorModel) renderMetrics(width int) string { + cpuPct := m.currentCPUPct() + mem := m.stats.Memory + + swapLabel := "n/a" + swapBar := barASCII(0, 24) + if mem.SwapTotalBytes > 0 { + swapLabel = fmt.Sprintf("%.1f%% (%s / %s)", mem.SwapUsedPct, humanBytes(mem.SwapUsedBytes), humanBytes(mem.SwapTotalBytes)) + swapBar = barASCII(mem.SwapUsedPct, 24) + } + + diskLine := "n/a" + diskBar := barASCII(0, 24) + if worst := worstDisk(m.stats.Disk); worst != nil { + diskLine = fmt.Sprintf("%.1f%% %s (%s)", worst.UsedPercent, worst.MountPoint, emptyFallback(worst.Health, "ok")) + diskBar = barASCII(worst.UsedPercent, 24) + } + + cardGap := 1 + cardW := (width - (cardGap * 3)) / 4 + if cardW < 20 { + cardW = 20 + } + + cpuCard := renderPanel("CPU", fmt.Sprintf("%.1f%%\n%s\nload %.2f %.2f %.2f", cpuPct, barASCII(cpuPct, 24), m.stats.CPU.Load1, m.stats.CPU.Load5, m.stats.CPU.Load15), cardW) + ramCard := renderPanel("RAM", fmt.Sprintf("%.1f%%\n%s\n%s / %s", mem.UsedPercent, barASCII(mem.UsedPercent, 24), humanBytes(mem.UsedBytes), humanBytes(mem.TotalBytes)), cardW) + swapCard := renderPanel("SWAP", fmt.Sprintf("%s\n%s", swapLabel, swapBar), cardW) + diskCard := renderPanel("DISK", fmt.Sprintf("%s\n%s", diskLine, diskBar), cardW) + + if width >= 110 { + return lipgloss.JoinHorizontal(lipgloss.Top, cpuCard, " ", ramCard, " ", swapCard, " ", diskCard) + } + + left := lipgloss.JoinVertical(lipgloss.Left, cpuCard, ramCard) + right := lipgloss.JoinVertical(lipgloss.Left, swapCard, diskCard) + return lipgloss.JoinHorizontal(lipgloss.Top, left, " ", right) +} + +func (m monitorModel) renderOverviewNetworkAndDisk(width int) string { + netW := width + diskW := width + if width >= 120 { + netW = int(math.Round(float64(width) * 0.62)) + diskW = width - netW - 1 + } + + netPanel := renderPanel("Network Top-5 (Current)", m.overviewNetworkBody(netW), netW) + diskPanel := renderPanel("Disk Health", m.diskBody(diskW), diskW) + + if width >= 120 { + return lipgloss.JoinHorizontal(lipgloss.Top, netPanel, " ", diskPanel) + } + return lipgloss.JoinVertical(lipgloss.Left, netPanel, diskPanel) +} + +func (m monitorModel) overviewNetworkBody(width int) string { + rows := m.networkRows() + if len(rows) == 0 { + return dimStyle.Render("no network interfaces reported") + } + + maxRows := 5 + if len(rows) < maxRows { + maxRows = len(rows) + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("%-2s %-12s %9s %9s %9s\n", "", "iface", "rx", "tx", "total")) + for i := 0; i < maxRows; i++ { + r := rows[i] + prefix := " " + if r.Interface == m.runtime.selectedIface { + prefix = ">" + } + b.WriteString(fmt.Sprintf("%-2s %-12s %9.2f %9.2f %9.2f\n", + prefix, + truncate(r.Interface, 12), + r.CurRxMbps, + r.CurTxMbps, + r.CurTotalMbps, + )) + } + if len(rows) > maxRows { + b.WriteString(dimStyle.Render(fmt.Sprintf("showing top-%d of %d, open Network view with TAB", maxRows, len(rows)))) + } else { + b.WriteString(dimStyle.Render("open Network view with TAB")) + } + return strings.TrimRight(b.String(), "\n") +} + +func (m monitorModel) diskBody(width int) string { + if !m.hasStats { + return dimStyle.Render("waiting for first metrics sample...") + } + if len(m.stats.Disk) == 0 { + return dimStyle.Render("no mounted disks reported") + } + + disks := append([]agent.DiskStats(nil), m.stats.Disk...) + sort.Slice(disks, func(i, j int) bool { + return disks[i].UsedPercent > disks[j].UsedPercent + }) + + maxRows := 8 + if len(disks) < maxRows { + maxRows = len(disks) + } + + var b strings.Builder + b.WriteString(fmt.Sprintf("%-12s %7s %-9s %s\n", "mount", "used", "health", "warnings")) + for i := 0; i < maxRows; i++ { + d := disks[i] + warns := "-" + if len(d.Warnings) > 0 { + warns = truncate(strings.Join(d.Warnings, "; "), max(8, width-36)) + } + health := emptyFallback(d.Health, "ok") + b.WriteString(fmt.Sprintf("%-12s %6.1f%% %-9s %s\n", + truncate(d.MountPoint, 12), + d.UsedPercent, + truncate(health, 9), + warns, + )) + } + if len(disks) > maxRows { + b.WriteString(dimStyle.Render(fmt.Sprintf("+%d more mounts", len(disks)-maxRows))) + } + return strings.TrimRight(b.String(), "\n") +} + +func (m monitorModel) renderClusterOverviewDashboard(width int) string { + panelH := m.mainViewportHeight(width) + clusters, activeID, err := m.svc.List() + if err != nil { + main, _ := renderFixedPanel(panelStyle, "Cluster Overview", "failed to load cluster inventory: "+err.Error(), width, panelH, m.contentScroll) + return main + } + + total := len(clusters) + agents := 0 + softwareDetected := 0 + for _, c := range clusters { + if c.Agent.Installed { + agents++ + } + if c.Software.Summary() != "-" && c.Software.Summary() != "none" { + softwareDetected++ + } + } + + activeName := "(none)" + for _, c := range clusters { + if c.ID == activeID { + activeName = c.Name + break + } + } + + telegramLine := "Telegram bot: " + m.telegramStatusSummary() + denom := total + if denom < 1 { + denom = 0 + } + summary := strings.Join([]string{ + fmt.Sprintf("Clusters total: %d", total), + fmt.Sprintf("Active cluster: %s", activeName), + fmt.Sprintf("Agent installed: %d/%d", agents, denom), + fmt.Sprintf("Software detected: %d/%d", softwareDetected, denom), + telegramLine, + }, "\n") + leftW, sideW := splitWidths(width) + + var tableBody string + if total == 0 { + tableBody = "No clusters connected yet.\nUse console (`t`) and run:\ncluster connect --name ... --host ... --user ..." + } else { + var table bytes.Buffer + tw := tabwriter.NewWriter(&table, 0, 2, 2, ' ', 0) + _, _ = fmt.Fprintln(tw, "ACTIVE\tNAME\tTARGET\tAGENT\tSOFTWARE\tUPDATED") + for _, c := range clusters { + active := "" + if c.ID == activeID { + active = "*" + } + agentState := "no" + if c.Agent.Installed { + agentState = fmt.Sprintf("yes:%d", c.Agent.Port) + } + target := truncate(fmt.Sprintf("%s@%s:%d", c.User, c.Host, c.Port), 24) + updated := c.UpdatedAt.Local().Format("01-02 15:04") + _, _ = fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", + active, + truncate(c.Name, 14), + target, + agentState, + truncate(c.Software.Summary(), 18), + updated, + ) + } + _ = tw.Flush() + tableBody = strings.TrimRight(table.String(), "\n") + } + + body := renderSectionsBody(leftW, []dashboardSection{ + {Title: "Cluster Overview", Body: summary}, + {Title: "Clusters", Body: tableBody}, + }) + main, _ := renderFixedPanel(panelStyle, "Clusters", body, leftW, panelH, m.contentScroll) + side := "" + if sideW > 0 { + side, _ = renderFixedPanel(panelStyle, "Tips & Activity", m.activityPanelBody(sideW), sideW, panelH, m.contentScroll) + } + return composeWithSidebar(main, side, width, leftW, sideW) +} + +func (m monitorModel) telegramStatusSummary() string { + if strings.TrimSpace(m.telegram.Token) == "" { + return "not configured" + } + state := "disabled" + if m.telegram.Enabled { + state = "enabled" + } + return fmt.Sprintf("%s, allowed users: %d", state, len(m.telegram.AllowedUserIDs)) +} + +func (m monitorModel) lockerStatusSummary() string { + if strings.TrimSpace(m.lockerHash) == "" { + return "not configured" + } + if m.lockerEnabled { + return "enabled (re-lock every 6h)" + } + return "disabled" +} + +func (m monitorModel) renderNetworkDashboard(width int) string { + rows := m.networkRows() + start, end, totalPages := paginationBounds(len(rows), m.page, m.pageSize) + pageRows := m.enrichNetworkRows(rows[start:end]) + leftW, sideW := splitWidths(width) + panelH := m.mainViewportHeight(width) + + meta := fmt.Sprintf("search=%q | page=%d/%d | total_ifaces=%d | range=%s", + m.searchTerm, max(1, m.page), max(1, totalPages), len(rows), m.historyRange.Label) + if strings.TrimSpace(m.pinnedIface) != "" { + meta += " | " + accentStyle.Render("locked="+m.pinnedIface) + } + meta += "\n" + m.metricTabs() + meta += "\nsearch: press `/`, Enter to lock/unlock current iface, Esc to cancel" + + var table strings.Builder + table.WriteString(fmt.Sprintf("%-2s %-12s %8s %8s %8s %8s %8s %8s %-18s\n", + "", "iface", "rx", "tx", "total", "avg", "peak", "used", "graph")) + for i, row := range pageRows { + marker := " " + if i == m.cursor { + marker = ">" + if row.Interface == m.pinnedIface { + marker = "*" + } + } + table.WriteString(fmt.Sprintf("%-2s %-12s %8.2f %8.2f %8.2f %8.2f %8.2f %8s %-18s\n", + marker, + truncate(row.Interface, 12), + row.CurRxMbps, + row.CurTxMbps, + row.CurTotalMbps, + row.AvgTotalMbps, + row.PeakTotal, + humanBitsRate(row.ConsumedByte), + truncate(row.Spark, 18), + )) + } + if len(pageRows) == 0 { + table.WriteString(dimStyle.Render("no interfaces matched current filter")) + } + + body := renderSectionsBody(leftW, []dashboardSection{ + {Title: "Network View", Body: meta}, + {Title: "Interfaces (paginated)", Body: strings.TrimRight(table.String(), "\n")}, + {Title: "Interface Details", Body: m.networkDetailsBody(pageRows, leftW)}, + }) + main, _ := renderFixedPanel(panelStyle, "Network", body, leftW, panelH, m.contentScroll) + side := "" + if sideW > 0 { + side, _ = renderFixedPanel(panelStyle, "Tips & Activity", m.activityPanelBody(sideW), sideW, panelH, m.contentScroll) + } + return composeWithSidebar(main, side, width, leftW, sideW) +} + +func (m monitorModel) renderSettingsDashboard(width int) string { + leftW, sideW := splitWidths(width) + panelH := m.mainViewportHeight(width) + + rows := []string{ + fmt.Sprintf("Refresh interval: %s", m.interval), + fmt.Sprintf("Network page size: %d", m.pageSize), + fmt.Sprintf("History range: %s", m.historyRange.Label), + fmt.Sprintf("Telegram bot: %s", m.telegramStatusSummary()), + fmt.Sprintf("TUI locker: %s", m.lockerStatusSummary()), + fmt.Sprintf("Locker password: %s", ternary(strings.TrimSpace(m.lockerHash) == "", "not set (Enter to set)", "set (Enter to replace)")), + } + + var body strings.Builder + body.WriteString("Use up/down to select setting, left/right (+/-) to change.\n") + body.WriteString("These settings are persisted to local config.\n\n") + for i, row := range rows { + prefix := " " + if i == m.settingsCursor { + prefix = ">" + } + body.WriteString(fmt.Sprintf("%s %s\n", prefix, row)) + } + body.WriteString("\n") + body.WriteString("Hints:\n") + body.WriteString("- press `tab` to cycle Overview -> Clusters -> Network -> Settings\n") + body.WriteString("- press `o` for Overview, `c` for Clusters, `n` for Network, `t` for Console\n") + body.WriteString("- select 'Locker password' and press Enter to set/replace password\n") + body.WriteString("- configure Telegram credentials in console:\n") + body.WriteString(" bot telegram set --token --allow --allow ") + + main, _ := renderFixedPanel(panelStyle, "Settings", strings.TrimRight(body.String(), "\n"), leftW, panelH, m.contentScroll) + side := "" + if sideW > 0 { + side, _ = renderFixedPanel(panelStyle, "Tips & Activity", m.activityPanelBody(sideW), sideW, panelH, m.contentScroll) + } + return composeWithSidebar(main, side, width, leftW, sideW) +} + +func (m monitorModel) docsViewportHeight() int { + h := m.height - 20 + if h < 10 { + h = 10 + } + return h +} + +func (m monitorModel) renderDocsDashboard(width int) string { + panelH := m.mainViewportHeight(width) + innerWidth := max(26, width-8) + lines := docsStyledLines(innerWidth) + viewport := m.docsViewportHeight() + total := len(lines) + offset := m.docsScroll + maxOffset := total - viewport + if maxOffset < 0 { + maxOffset = 0 + } + if offset < 0 { + offset = 0 + } + if offset > maxOffset { + offset = maxOffset + } + end := offset + viewport + if end > total { + end = total + } + view := append([]string(nil), lines[offset:end]...) + for len(view) < viewport { + view = append(view, "") + } + progress := "top" + if total > 0 { + progress = fmt.Sprintf("lines %d-%d/%d", offset+1, end, total) + } + meta := dimStyle.Render("scroll: ↑/↓ or j/k, PgUp/PgDn, Home/End") + " · " + brightStyle.Render(progress) + body := meta + "\n\n" + strings.Join(view, "\n") + panel, _ := renderFixedPanel(panelStyle, "Docs", body, width, panelH, 0) + return panel +} + +func docsStyledLines(width int) []string { + raw := docsRawLines() + out := make([]string, 0, len(raw)) + docTitle := lipgloss.NewStyle().Bold(true).Foreground(ccAccent) + docSection := lipgloss.NewStyle().Bold(true).Foreground(ccTextBright) + docSubsection := lipgloss.NewStyle().Bold(true).Foreground(ccAccent) + docCode := lipgloss.NewStyle().Foreground(ccTextBright) + for _, line := range raw { + trimmed := strings.TrimSpace(line) + switch { + case strings.HasPrefix(line, "### "): + out = append(out, docSubsection.Render(strings.TrimPrefix(line, "### "))) + case strings.HasPrefix(line, "## "): + out = append(out, docSection.Render(strings.TrimPrefix(line, "## "))) + case strings.HasPrefix(line, "# "): + out = append(out, docTitle.Render(strings.TrimPrefix(line, "# "))) + case strings.HasPrefix(line, " "): + out = append(out, softStyle.Render(truncate(line, width))) + case strings.HasPrefix(trimmed, "`") && strings.HasSuffix(trimmed, "`"): + code := strings.TrimSuffix(strings.TrimPrefix(trimmed, "`"), "`") + out = append(out, docCode.Render(code)) + case strings.HasPrefix(trimmed, "- "): + out = append(out, "• "+strings.TrimPrefix(trimmed, "- ")) + case trimmed == "": + out = append(out, "") + default: + out = append(out, truncate(line, width)) + } + } + return out +} + +func docsRawLines() []string { + return []string{ + "# PXmon (Phylex Monitor) Docs", + "Fullscreen command reference with command variants and examples.", + "", + "## Navigation", + "- Open docs page: `d` in TUI or `docs` in command console.", + "- Scroll: `↑/↓`, `j/k`, `PgUp/PgDn`, `Home/End`.", + "- Cycle views: `tab`.", + "- Quick jump: `o` overview, `c` clusters, `n` network, `s` settings.", + "", + "## Root Commands", + "`pxmon cluster ...`", + "`pxmon tui`", + "`pxmon clusters`", + "`pxmon network`", + "`pxmon bot telegram ...`", + "`pxmon locker ...`", + "`pxmon config export|import ...`", + "`pxmon explain [--find ]`", + "", + "## 🧭 Cluster Command Map", + "`pxmon cluster connect|add ...`", + "`pxmon cluster list|ls`", + "`pxmon cluster show|get [cluster]`", + "`pxmon cluster current`", + "`pxmon cluster use `", + "`pxmon cluster disconnect|remove|rm `", + "`pxmon cluster set-auth|auth|password|passwd ...`", + "`pxmon cluster openssh|ssh [cluster]`", + "`pxmon cluster ping|check [cluster] [--agent]`", + "`pxmon cluster bootstrap [cluster]`", + "`pxmon cluster agent status|update|versions [cluster]`", + "`pxmon cluster stats [cluster] [--once]`", + "`pxmon cluster usage [cluster] --range `", + "`pxmon cluster traffic [cluster] --range <1h|1d|1mo|all>`", + "`pxmon cluster graph [cluster] --range <1d|1mo|all> [--iface ] [--out ]`", + "`pxmon cluster p95 [cluster] --iface --range `", + "", + "## 🔐 Connect Cluster Variants", + "### SSH key auth", + "`pxmon cluster connect --name eu-1 --host 10.0.0.10 --user root --auth key --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/.ssh/id_ed25519.pxmonpassphrase`", + "### Password auth (store password)", + "`pxmon cluster connect --name eu-2 --host 10.0.0.20 --user root --auth password --password 'secret' --store-password`", + "### ipfabric transport + password auth", + "`pxmon cluster connect --name vm-host --host 198.51.100.30 --user root --type ipfabric --auth password --password 'example-pass' --store-password`", + "### Insecure host key mode", + "`pxmon cluster connect --name lab-1 --host 10.0.0.30 --user root --auth key --key-path ~/.ssh/id_ed25519 --insecure-host-key`", + "### Set active cluster", + "`pxmon cluster use eu-1`", + "### Connectivity checks", + "`pxmon cluster list`", + "`pxmon cluster ping eu-1`", + "`pxmon cluster ping eu-1 --agent`", + "", + "## Auth Update Variants", + "### Switch to password auth", + "`pxmon cluster set-auth eu-1 --auth password --password 'secret' --store-password`", + "### Switch to key auth", + "`pxmon cluster set-auth eu-1 --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/Desktop/passphrase.txt --store-key-passphrase`", + "### Change transport mode", + "`pxmon cluster set-auth eu-1 --type ipfabric`", + "", + "## Agent Operations", + "`pxmon cluster bootstrap eu-1`", + "`pxmon cluster agent status`", + "`pxmon cluster agent status eu-1`", + "`pxmon cluster agent update eu-1 --restart-bot=true`", + "`pxmon cluster agent versions`", + "", + "## 🧩 Software Detection", + "`pxmon cluster software show [cluster]`", + "`pxmon cluster software scan [cluster]`", + "", + "## 📈 Metrics and Network", + "### Snapshot stats", + "`pxmon cluster stats eu-1 --once`", + "`pxmon cluster usage eu-1 --range 1d`", + "`pxmon cluster traffic eu-1 --range 30d`", + "`pxmon cluster p95 eu-1 --iface eth0 --range 30d`", + "`pxmon cluster p95 eu-1 --iface eth0 --range 30d --graph`", + "`pxmon cluster p95 eu-1 --iface vm2151_net0 --range 30d --graph`", + "`pxmon cluster graph eu-1 --range 1d --iface vm2151_net0 --out vm2151_net0-1d.png`", + "", + "## 🚨 Alert Rules", + "### Show / set base thresholds", + "`pxmon cluster alert show [cluster]`", + "`pxmon cluster alert set eu-1 --cpu 85 --ram 90 --disk 90 --net-mbps 300`", + "### Sustained network rule variants", + "`pxmon cluster alert set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-cooldown-mins 30 --net-sustain-include net0`", + "`pxmon cluster alert set eu-1 --net-sustain-iface vm2151_net0 --net-sustain-mbps 500 --net-sustain-mins 60`", + "`pxmon cluster alert set eu-1 --net-sustain-enabled=true --net-sustain-include net0 --net-sustain-exclude backup`", + "", + "## Alert Routing", + "`pxmon cluster alert-routing show [cluster]`", + "`pxmon cluster alert-routing set eu-1 --critical-immediate=true --warning-batch-mins 5`", + "", + "## VM Alerts", + "`pxmon cluster alert-vm show [cluster]`", + "`pxmon cluster alert-vm set eu-1 --enabled --warn-on-shutoff --min-running 118`", + "`pxmon cluster alert-vm check eu-1`", + "", + "## 🏷️ Tags", + "### Cluster tags", + "`pxmon cluster tag add eu-1 --tags prod,billing`", + "`pxmon cluster tag rm eu-1 --tags billing`", + "`pxmon cluster tag ls eu-1`", + "### VM tags", + "`pxmon cluster kvm-tag add eu-1 --vm vm2151 --tags critical,net-heavy`", + "`pxmon cluster kvm-tag rm eu-1 --vm vm2151 --tags net-heavy`", + "`pxmon cluster kvm-tag ls eu-1 --vm vm2151`", + "", + "## 💾 Backups", + "### Target management", + "`pxmon cluster backup target ls`", + "### Add SFTP target", + "`pxmon cluster backup target add --name sftp1 --type sftp --sftp-host backup.example.net --sftp-user backup --sftp-password '***' --sftp-base /pxmon`", + "### Add S3 target", + "`pxmon cluster backup target add --name s3-main --type s3 --s3-endpoint s3.example.net --s3-region us-east-1 --s3-bucket backups --s3-prefix pxmon --s3-access-key AKIA... --s3-secret-key ... --s3-ssl=true --s3-path-style=true`", + "### Test / remove target", + "`pxmon cluster backup target test sftp1`", + "`pxmon cluster backup target rm sftp1`", + "### Create backup plan", + "`pxmon cluster backup plan add --name vm-images --cluster eu-1 --target sftp1 --path /var/lib/libvirt/images --every 6h`", + "`pxmon cluster backup plan add --name etc-backup --cluster eu-1 --target s3-main --path /etc --path /opt/app/config --every 24h --retain-days 30 --schedule=true`", + "### Plan list / remove / run", + "`pxmon cluster backup plan ls`", + "`pxmon cluster backup plan rm vm-images`", + "`pxmon cluster backup plan run vm-images`", + "`pxmon cluster backup run vm-images`", + "", + "## Drift and Baseline", + "`pxmon cluster drift [cluster]`", + "`pxmon cluster drift baseline set [cluster]`", + "`pxmon cluster drift baseline show [cluster]`", + "`pxmon cluster drift ack [cluster] --kind baseline_software --for 24h`", + "", + "## 📚 Runbooks", + "`pxmon cluster runbook list`", + "`pxmon cluster runbook show vm-health-check`", + "`pxmon cluster runbook run vm-health-check`", + "`pxmon cluster runbook rm custom-1`", + "### Add runbook variants", + "`pxmon cluster runbook add --id custom-1 --name 'Custom' --step 'Agent|cluster agent status' --step 'Drift|cluster drift'`", + "`pxmon cluster runbook add --edit`", + "`pxmon cluster runbook add --from /tmp/pxmon-runbook.json`", + "", + "## Runbook Triggers", + "`pxmon cluster runbook-trigger show [cluster]`", + "`pxmon cluster runbook-trigger set eu-1 --enabled --on-vm-shutoff --runbook-id vm-health-check --cooldown-mins 30`", + "", + "## ⏱️ Scheduler", + "`pxmon cluster schedule ls`", + "`pxmon cluster schedule add --name audit --mode observer --cmd 'cluster drift eu-1' --every 30m --backoff 30s --jitter-sec 5 --retry-max 3`", + "`pxmon cluster schedule add --name mk --cluster eu-1 --mode shell --cmd 'mkdir -p /tmp/test' --every 10m`", + "`pxmon cluster schedule add --edit`", + "`pxmon cluster schedule add --from /tmp/pxmon-schedule.json`", + "`pxmon cluster schedule rm audit`", + "`pxmon cluster schedule run-due`", + "`pxmon cluster schedule start --interval 30s`", + "`pxmon cluster schedule stop`", + "`pxmon cluster schedule status`", + "`pxmon cluster schedule logs --tail 200`", + "`pxmon cluster schedule worker --interval 30s`", + "", + "## 📦 Reports, SLO, Capacity, Changes", + "`pxmon cluster report export --format json --out ./cluster-report.json`", + "`pxmon cluster report export --format csv --out ./cluster-report.csv`", + "`pxmon cluster slo eu-1 --range 30d`", + "`pxmon cluster capacity forecast eu-1 --range 30d`", + "`pxmon cluster change-history --tail 100`", + "", + "## Telegram Bot", + "`pxmon bot telegram show`", + "`pxmon bot telegram set --token --allow 123456789 --allow 987654321`", + "`pxmon bot telegram run`", + "", + "## Config Export and Import", + "`pxmon config export --out ./pxmon-export.enc`", + "`pxmon config import ./pxmon-export.enc`", + "", + "## TUI Console Built-ins", + "`help`, `history`, `clear`, `overview`, `clusters`, `network`, `settings`, `docs`, `quit`", + "`console full`, `console dock`, `console toggle`", + "", + "## Backup Connection Model", + "- Archive is built on remote cluster over SSH (`tar` stream).", + "- Upload to SFTP/S3 is performed from the machine running `pxmon`.", + "", + "## Note", + "This page focuses on command variants and examples.", + "Use `pxmon cluster help` for short quick-help output.", + } +} + +func (m monitorModel) networkDetailsBody(pageRows []networkRow, width int) string { + if len(pageRows) == 0 { + return dimStyle.Render("no interface selected") + } + if m.cursor < 0 || m.cursor >= len(pageRows) { + return dimStyle.Render("no interface selected") + } + row := pageRows[m.cursor] + points := m.history[row.Interface] + if len(points) == 0 { + return fmt.Sprintf("Interface: %s\nNo history points yet. Wait a few refresh cycles.", row.Interface) + } + + series := metricSeries(points, m.metric) + cur := metricCurrent(row, m.metric) + stats := computeMetricStats(points, m.metric) + last24h := metricUsageBytesSince(points, m.metric, time.Now().UTC().Add(-24*time.Hour)) + last30d := metricUsageBytesSince(points, m.metric, time.Now().UTC().Add(-30*24*time.Hour)) + graphW := max(20, width-12) + graphH := 10 + var b strings.Builder + ifaceLabel := row.Interface + if row.Interface == m.pinnedIface { + ifaceLabel += " " + accentStyle.Render("[locked]") + } + b.WriteString(fmt.Sprintf("Interface: %s\n", ifaceLabel)) + b.WriteString(fmt.Sprintf("Metric: %s | Current: %.2f Mbps | Samples: %d\n", metricLabel(m.metric), cur, stats.Samples)) + b.WriteString(fmt.Sprintf("Usage 24h: %s | Usage 30d: %s\n", humanBytes(last24h), humanBytes(last30d))) + b.WriteString(fmt.Sprintf("Peak: %.2f Mbps | Peak hold: %s\n", stats.PeakMbps, stats.PeakDuration.Truncate(time.Second))) + b.WriteString(renderBigGraphASCII(series, graphW, graphH)) + return strings.TrimRight(b.String(), "\n") +} + +func (m monitorModel) renderAlerts(width int) string { + alerts := collectAlerts(m.stats, &m.runtime) + if len(alerts) == 0 { + return renderPanel("Alerts", okStyle.Render("none"), width) + } + + var b strings.Builder + maxRows := 7 + if len(alerts) < maxRows { + maxRows = len(alerts) + } + for i := 0; i < maxRows; i++ { + b.WriteString("- ") + b.WriteString(truncate(alerts[i], width-8)) + b.WriteByte('\n') + } + if len(alerts) > maxRows { + b.WriteString(dimStyle.Render(fmt.Sprintf("+%d more", len(alerts)-maxRows))) + } + return renderPanel("Alerts", strings.TrimRight(b.String(), "\n"), width) +} + +func (m monitorModel) renderBottomLine(width int) string { + prompt := "" + switch m.inputMode { + case inputSearch: + prompt = "/" + m.inputBuf + case inputLockerNew, inputLockerConfirm: + prompt = "locker password: " + strings.Repeat("*", len([]rune(m.inputBuf))) + } + + help := "keys: t console | ctrl+t fullscreen console | / search(network) | tab switch view | d docs | c clusters | u usage | s settings | r refresh | q quit" + if m.view == viewOverview { + help = "keys: left/right iface | tab switch view | d docs | c clusters | u usage | s settings | t console | ctrl+t fullscreen | r refresh | q quit" + } else if m.view == viewClusters { + help = "keys: tab switch view | o overview | n network | u usage | s settings | d docs | t console | ctrl+t fullscreen | r refresh | q quit" + } else if m.view == viewNetwork { + help = "keys: up/down row | left/right or 1/2/3 metric | n/p page | / search | d docs | u usage | c clusters | s settings | t console | ctrl+t fullscreen | r refresh | q quit" + } else if m.view == viewSettings { + help = "keys: up/down setting | left/right adjust | tab switch view | d docs | c clusters | u usage | t console | ctrl+t fullscreen | q quit" + } else if m.view == viewDocs { + help = "keys: ↑/↓ or j/k scroll | PgUp/PgDn | Home/End | o/c/n/s switch view | tab switch | t console | q quit" + } else if m.view == viewUsage { + help = "keys: 1 live | 2 1h | 3 1d | 4 1mo | 5 all | r refresh | o back | t console | ctrl+t fullscreen | q quit" + } + if m.termMode { + help = "console mode: PXmon (Phylex Monitor) commands | console full/dock/toggle | end line with \\ to continue | alt+←/→ word move | ! shell escape | ctrl+g or ctrl+t back" + } + if m.inputMode == inputSearch { + help = "search mode: type query text | Enter apply | Esc cancel" + } else if m.inputMode == inputLockerNew { + help = "locker setup: type new password | Enter continue | Esc cancel" + } else if m.inputMode == inputLockerConfirm { + help = "locker setup: confirm password | Enter save | Esc cancel" + } + + parts := []string{} + if prompt != "" { + parts = append(parts, prompt) + } + if m.statusMsg != "" { + parts = append(parts, m.statusMsg) + } else { + parts = append(parts, help) + } + line := strings.Join(parts, " · ") + hint := softStyle.Render("?") + dimStyle.Render(" for shortcuts") + body := " " + hint + " " + dimStyle.Render("·") + " " + dimStyle.Render(truncate(line, max(10, width-24))) + return body +} + +func (m monitorModel) renderConsoleInput() string { + prompt := accentStyle.Render("> ") + if strings.TrimSpace(m.termCont) != "" { + prompt = warnStyle.Render("CONT> ") + } + runes := []rune(m.termPartial) + cursor := m.termCursor + if cursor < 0 { + cursor = 0 + } + if cursor > len(runes) { + cursor = len(runes) + } + + // Keep cursor editing stable for mid-line cursor positions; highlight when + // the cursor is at end (common case) to avoid ANSI width/cursor drift. + if cursor == len(runes) { + line := highlightConsoleInput(m.termPartial) + return prompt + line + lipgloss.NewStyle().Reverse(true).Render(" ") + } + left := string(runes[:cursor]) + cur := string(runes[cursor]) + right := string(runes[cursor+1:]) + return prompt + left + lipgloss.NewStyle().Reverse(true).Render(cur) + right +} + +func (m monitorModel) mainViewportHeight(width int) int { + if m.height <= 0 { + return 18 + } + header := m.renderHeader(width) + dock := m.renderCommandDock(width) + h := m.contentBudget(header, dock) + if h < 8 { + h = 8 + } + return h +} + +func (m monitorModel) submitConsoleLine() (tea.Model, tea.Cmd) { + raw := m.termPartial + line := strings.TrimSpace(raw) + + m.termPartial = "" + m.termCursor = 0 + m.termHistPos = -1 + m.termDraft = "" + + if line == "" && strings.TrimSpace(m.termCont) != "" { + line = strings.TrimSpace(m.termCont) + m.termCont = "" + } + if line == "" { + return m, nil + } + + normalizedLine := strings.Join(strings.Fields(line), " ") + now := time.Now() + if normalizedLine == m.termLastSubmit && now.Sub(m.termLastSubmitAt) < 2500*time.Millisecond { + m.setStatus("ignored duplicate command") + return m, nil + } + m.termLastSubmit = normalizedLine + m.termLastSubmitAt = now + + if strings.HasSuffix(strings.TrimRight(line, " \t"), "\\") { + part := strings.TrimSpace(strings.TrimSuffix(strings.TrimRight(line, " \t"), "\\")) + if part != "" { + if strings.TrimSpace(m.termCont) == "" { + m.termCont = part + } else { + m.termCont += " " + part + } + } + m.pushTerminalLine(accentStyle.Bold(true).Render("… ") + softStyle.Render(part+" \\")) + m.setStatus("line continued (finish command and press Enter)") + return m, nil + } + if strings.TrimSpace(m.termCont) != "" { + line = strings.TrimSpace(m.termCont + " " + line) + m.termCont = "" + } + + m.pushTerminalLine(accentStyle.Bold(true).Render("> ") + brightStyle.Render(line)) + m.pushConsoleHistory(line) + + normalized := strings.Join(strings.Fields(strings.ToLower(line)), " ") + switch normalized { + case "clear", "cls": + m.termLines = nil + m.setStatus("console cleared") + return m, nil + case "history": + if len(m.termHistory) == 0 { + m.pushTerminalLine("history is empty") + return m, nil + } + for i, h := range m.termHistory { + m.pushTerminalLine(fmt.Sprintf("%3d %s", i+1, h)) + } + return m, nil + case "help", "?": + m.appendConsoleOutput(consoleHelpText()) + return m, nil + case "exit", "quit": + m.termMode = false + m.termFull = false + m.setStatus("console hidden") + return m, nil + case "privacy": + m.privacyMode = !m.privacyMode + if m.privacyMode { + m.pushTerminalLine(okStyle.Render("privacy mode: ON") + dimStyle.Render(" (IPs, hostnames, tokens, secrets are redacted)")) + } else { + m.pushTerminalLine(warnStyle.Render("privacy mode: OFF")) + } + return m, nil + case "settings", "config": + m.view = viewSettings + m.setStatus("opened settings") + return m, nil + case "overview": + m.view = viewOverview + m.setStatus("opened overview") + return m, nil + case "clusters", "cluster-overview": + m.view = viewClusters + m.setStatus("opened cluster overview") + return m, nil + case "network": + m.view = viewNetwork + m.setStatus("opened network") + return m, nil + case "docs", "manual": + m.view = viewDocs + m.setStatus("opened docs") + return m, nil + case "console full", "console fullscreen": + m.termFull = true + m.setStatus("console fullscreen") + return m, nil + case "console dock": + m.termFull = false + m.setStatus("console docked") + return m, nil + case "console toggle": + m.termFull = !m.termFull + if m.termFull { + m.setStatus("console fullscreen") + } else { + m.setStatus("console docked") + } + return m, nil + } + + if strings.HasPrefix(line, "!") { + shellLine := strings.TrimSpace(strings.TrimPrefix(line, "!")) + if shellLine == "" { + m.pushTerminalLine("usage: !") + return m, nil + } + m.termBusy = true + m.thinkFrame = 0 + m.spinnerStart = time.Now() + m.termExecSeq++ + m.termExecActive = m.termExecSeq + m.setStatus("running shell escape") + cmds := []tea.Cmd{m.runShellEscapeCmd(shellLine, m.termExecActive)} + if !m.spinnerActive { + m.spinnerActive = true + cmds = append(cmds, spinnerTickCmd()) + } + return m, tea.Batch(cmds...) + } + + if parsed, perr := parseShellArgs(line); perr == nil && len(parsed) > 0 { + norm := normalizeObserverConsoleArgs(parsed) + if len(norm) >= 2 && strings.EqualFold(norm[0], "cluster") && + (strings.EqualFold(norm[1], "openssh") || strings.EqualFold(norm[1], "ssh")) { + return m.startSSHSessionCmd(line, norm[2:]) + } + // Live streaming for plugin commands: `kvm top --live`, + // `frr bgp --live`, etc. Opens a fullscreen dashboard that + // re-runs the command on a tick and replaces the buffer. + if spec, ok := detectLivePluginInvocation(parsed); ok { + return m.startLiveCmd(spec) + } + } + + m.termBusy = true + m.thinkFrame = 0 + m.spinnerStart = time.Now() + m.termExecSeq++ + m.termExecActive = m.termExecSeq + m.setStatus("running command") + cmds := []tea.Cmd{m.runObserverConsoleCmd(line, m.termExecActive)} + if !m.spinnerActive { + m.spinnerActive = true + cmds = append(cmds, spinnerTickCmd()) + } + return m, tea.Batch(cmds...) +} + +func (m monitorModel) startSSHSessionCmd(originalLine string, args []string) (tea.Model, tea.Cmd) { + selector := "" + if len(args) > 0 { + first := strings.TrimSpace(args[0]) + if first != "" && !strings.HasPrefix(first, "-") { + selector = first + } + } + + c, err := m.svc.Get(selector) + if err != nil { + m.appendConsoleOutput("openssh: " + err.Error()) + m.setStatus("openssh: " + err.Error()) + return m, nil + } + + banner := fmt.Sprintf("dialing ssh: %s@%s:%d", c.User, c.Host, c.Port) + m.appendConsoleOutput(banner) + m.setStatus(banner) + + // Leave console mode so the embedded view owns the screen. + m.termMode = false + m.termFull = false + + return m, m.startEmbeddedSSHCmd(c.ID) +} + +func (m monitorModel) runObserverConsoleCmd(line string, id int64) tea.Cmd { + configPath := m.svc.ConfigPath() + return func() tea.Msg { + out, code := runObserverScopedCommand(m.svc, configPath, line, observerCommandOptions{ + AllowShellEscape: false, + StatsAutoOnce: false, + BlockBotRun: false, + EmbeddedConsole: true, + }) + return consoleExecResultMsg{ + ID: id, + Command: line, + Output: out, + ExitCode: code, + } + } +} + +func (m monitorModel) runPluginConsole(args []string) (bool, string, int) { + return runPluginArgs(m.svc, args) +} + +func parseClusterSelectorArg(args []string) (string, []string, error) { + if len(args) == 0 { + return "", nil, nil + } + + out := make([]string, 0, len(args)) + selector := "" + for i := 0; i < len(args); i++ { + item := strings.TrimSpace(args[i]) + switch { + case item == "--cluster" || item == "-c": + if i+1 >= len(args) { + return "", nil, errors.New("missing value for --cluster") + } + selector = strings.TrimSpace(args[i+1]) + i++ + case strings.HasPrefix(item, "--cluster="): + selector = strings.TrimSpace(strings.TrimPrefix(item, "--cluster=")) + default: + out = append(out, args[i]) + } + } + + return selector, out, nil +} + +func (m monitorModel) runShellEscapeCmd(line string, id int64) tea.Cmd { + cmdline := strings.TrimSpace(line) + return func() tea.Msg { + cmd := exec.Command("/bin/sh", "-lc", cmdline) + out, err := cmd.CombinedOutput() + code := 0 + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + code = exitErr.ExitCode() + } else { + code = 1 + if len(out) == 0 { + out = []byte(err.Error()) + } + } + } + return consoleExecResultMsg{ + ID: id, + Command: "!" + cmdline, + Output: string(out), + ExitCode: code, + } + } +} + +func runObserverCommand(configPath string, args []string, embeddedConsole bool) (string, int) { + if len(args) == 0 { + return "", 0 + } + + switch strings.ToLower(strings.TrimSpace(args[0])) { + case "shell", "tui", "network", "clusters": + return "interactive screens are already open in TUI; use UI hotkeys", 2 + case "bot": + if len(args) >= 3 && strings.EqualFold(args[1], "telegram") && strings.EqualFold(args[2], "run") { + return "run `bot telegram run` in a standalone terminal session", 2 + } + case "cluster": + if len(args) > 1 && strings.EqualFold(args[1], "stats") && !hasArg(args[2:], "--once") { + return "use UI overview/network for realtime stats; use `cluster stats --once` for snapshot", 2 + } + if len(args) > 1 && (strings.EqualFold(args[1], "openssh") || strings.EqualFold(args[1], "ssh")) { + return "run `cluster openssh` directly from the console — it requires an interactive terminal", 2 + } + } + + var outBuf bytes.Buffer + var errBuf bytes.Buffer + runner := New(&outBuf, &errBuf) + prevEmbedded := os.Getenv("PXMON_EMBEDDED_CONSOLE") + if embeddedConsole { + _ = os.Setenv("PXMON_EMBEDDED_CONSOLE", "1") + } + defer func() { + if embeddedConsole { + if prevEmbedded == "" { + _ = os.Unsetenv("PXMON_EMBEDDED_CONSOLE") + } else { + _ = os.Setenv("PXMON_EMBEDDED_CONSOLE", prevEmbedded) + } + } + }() + code := runner.runRootCommand(args, configPath, false, false) + + out := strings.TrimSpace(outBuf.String()) + errText := strings.TrimSpace(errBuf.String()) + switch { + case out != "" && errText != "": + return out + "\n" + errText, code + case errText != "": + return errText, code + default: + return out, code + } +} + +func normalizeObserverConsoleArgs(args []string) []string { + if len(args) == 0 { + return nil + } + if strings.EqualFold(args[0], "pxmon") || strings.EqualFold(args[0], "pxmon") { + args = args[1:] + } + if len(args) == 0 { + return nil + } + + head := strings.ToLower(strings.TrimSpace(args[0])) + switch head { + case "qvmtop": + return []string{"kvm", "top"} + case "qvmtoplive": + return []string{"kvm", "top", "--live"} + case "qvmnettop": + return []string{"kvm", "net-top"} + case "qvmnettoplive": + return []string{"kvm", "net-top", "--live"} + case "alert", "alerts": + if len(args) == 1 { + return []string{"cluster", "alert", "show"} + } + return append([]string{"cluster", "alert"}, args[1:]...) + case "software", "plugins": + return append([]string{"cluster", "software"}, args[1:]...) + case "clusters", "nodes": + return []string{"cluster", "list"} + case "connect": + if len(args) == 2 && !strings.HasPrefix(args[1], "-") { + return []string{"cluster", "use", args[1]} + } + return append([]string{"cluster", "connect"}, args[1:]...) + case "ping", "check", "list", "ls", "show", "get", "current", "use", + "bootstrap", "stats", "disconnect", "remove", "rm", "openssh", "ssh", + "set-auth", "password", "passwd", "usage", "traffic", "graph": + return append([]string{"cluster"}, args...) + case "cluster": + if len(args) == 3 && strings.EqualFold(args[1], "connect") && !strings.HasPrefix(args[2], "-") { + return []string{"cluster", "use", args[2]} + } + return args + default: + return args + } +} + +func hasArg(args []string, token string) bool { + for _, a := range args { + if strings.EqualFold(strings.TrimSpace(a), token) || strings.HasPrefix(strings.ToLower(a), strings.ToLower(token)+"=") { + return true + } + } + return false +} + +func consoleHelpText() string { + return strings.Join([]string{ + "Observer Console commands:", + " cluster list", + " cluster use ", + " cluster ping [--agent]", + " cluster connect --name eu-1 --host 10.0.0.10 --port 22 --user root --auth key --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/.ssh/id_ed25519.pxmonpassphrase", + " cluster connect --name vm-host --host 1.2.3.4 --user root --type ipfabric --auth password --password 'pw' --store-password", + " cluster bootstrap ", + " cluster openssh (alias: cluster ssh) — interactive SSH in this TUI", + " cluster set-auth --auth password --password 'example-pass' --store-password", + " cluster set-auth --key-path ~/.ssh/id_ed25519 --key-passphrase-file ~/Desktop/passphrase.txt --store-key-passphrase", + " cluster set-auth --type ipfabric (enable SSH-tunneled agent calls)", + " cluster repo-tunnel install --gateway --table
-- dnf install -y curl jq", + " cluster usage [name] [--range live|1h|1d|1mo|all] [--du /] (billing + top procs/folders)", + " cluster traffic [name] [--range 1h|1d|1mo|all] (P95 text summary)", + " cluster graph [name] [--range 1d|1mo|all] [--out file.png] (PNG chart with P95 line)", + " docs (open full docs page)", + " console full | console dock | console toggle (Command Console layout)", + " config export ./backup.enc (encrypted passphrase-protected bundle)", + " config import ./backup.enc [--replace]", + " alerts set --net-mbps 300 --ram 90 --disk 90", + " alerts set --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup", + " cluster software scan ", + " bot telegram show", + " bot telegram set --token --allow --allow ", + "", + "Aliases:", + " ping -> cluster ping ", + " alerts ... -> cluster alert ...", + " connect -> cluster use ", + " software ... -> cluster software ...", + "", + "Plugin commands (SSH on selected cluster):", + " kvm list | kvm start | kvm stop | kvm reboot ", + " kvm top | kvm net-top", + " lxc list | lxc start | lxc stop | lxc restart | lxc stats ", + " lxc top | lxc net-top", + " lxd list | lxd start | lxd stop | lxd restart | lxd stats ", + " lxd top | lxd net-top", + " bird status | bird protocols | bird routes", + " frr status | frr routes | frr bgp | frr ospf", + " add --cluster to run on non-active cluster", + "", + "Built-ins:", + " help, history, clear, overview, clusters, network, settings, docs, quit", + " line continuation: end line with \\; press Enter on empty line to run", + " edit: alt+left/right (or alt+b/alt+f) moves by word", + "", + "Shell escape (secondary):", + " !ls -la", + " !ssh root@host", + }, "\n") +} + +func (m *monitorModel) appendConsoleOutput(text string) { + if strings.TrimSpace(text) == "" { + return + } + clean := sanitizeTerminalChunk(text) + clean = strings.ReplaceAll(clean, "\r\n", "\n") + clean = strings.ReplaceAll(clean, "\r", "\n") + raw := strings.Split(clean, "\n") + for len(raw) > 0 && strings.TrimSpace(raw[len(raw)-1]) == "" { + raw = raw[:len(raw)-1] + } + branch := softStyle.Render(" ⎿ ") + indent := " " + for i, line := range raw { + if i == 0 { + m.pushTerminalLine(branch + line) + } else { + m.pushTerminalLine(indent + line) + } + } +} + +func (m *monitorModel) scrollConsole(delta int) { + m.termScroll += delta + m.clampTermScroll() +} + +func (m *monitorModel) clampTermScroll() { + maxOffset := len(m.termLines) - m.dockOutputHeight() + if maxOffset < 0 { + maxOffset = 0 + } + if m.termScroll > maxOffset { + m.termScroll = maxOffset + } + if m.termScroll < 0 { + m.termScroll = 0 + } +} + +func (m *monitorModel) pushConsoleHistory(line string) { + line = strings.TrimSpace(line) + if line == "" { + return + } + if len(m.termHistory) > 0 && m.termHistory[len(m.termHistory)-1] == line { + return + } + m.termHistory = append(m.termHistory, line) + if len(m.termHistory) > 200 { + m.termHistory = m.termHistory[len(m.termHistory)-200:] + } + m.termHistPos = -1 + m.termDraft = "" +} + +func (m *monitorModel) historyUp() { + if len(m.termHistory) == 0 { + return + } + if m.termHistPos == -1 { + m.termDraft = m.termPartial + m.termHistPos = len(m.termHistory) - 1 + } else if m.termHistPos > 0 { + m.termHistPos-- + } + m.termPartial = m.termHistory[m.termHistPos] + m.termCursor = len([]rune(m.termPartial)) +} + +func (m *monitorModel) historyDown() { + if len(m.termHistory) == 0 || m.termHistPos == -1 { + return + } + if m.termHistPos < len(m.termHistory)-1 { + m.termHistPos++ + m.termPartial = m.termHistory[m.termHistPos] + } else { + m.termHistPos = -1 + m.termPartial = m.termDraft + } + m.termCursor = len([]rune(m.termPartial)) +} + +func (m *monitorModel) moveConsoleCursor(delta int) { + size := len([]rune(m.termPartial)) + m.termCursor += delta + if m.termCursor < 0 { + m.termCursor = 0 + } + if m.termCursor > size { + m.termCursor = size + } +} + +func (m *monitorModel) moveConsoleWord(dir int) { + r := []rune(m.termPartial) + n := len(r) + c := m.termCursor + if c < 0 { + c = 0 + } + if c > n { + c = n + } + if dir < 0 { + i := c - 1 + for i >= 0 && (r[i] == ' ' || r[i] == '\t') { + i-- + } + for i >= 0 && r[i] != ' ' && r[i] != '\t' { + i-- + } + m.termCursor = i + 1 + return + } + i := c + for i < n && (r[i] == ' ' || r[i] == '\t') { + i++ + } + for i < n && r[i] != ' ' && r[i] != '\t' { + i++ + } + m.termCursor = i +} + +func (m *monitorModel) insertConsoleText(text string) { + left := []rune(m.termPartial) + cursor := m.termCursor + if cursor < 0 { + cursor = 0 + } + if cursor > len(left) { + cursor = len(left) + } + + inserted := []rune(text) + merged := make([]rune, 0, len(left)+len(inserted)) + merged = append(merged, left[:cursor]...) + merged = append(merged, inserted...) + merged = append(merged, left[cursor:]...) + + m.termPartial = string(merged) + m.termCursor = cursor + len(inserted) + m.termHistPos = -1 +} + +func (m *monitorModel) deleteConsolePrev() { + r := []rune(m.termPartial) + if m.termCursor <= 0 || len(r) == 0 { + return + } + cursor := m.termCursor + if cursor > len(r) { + cursor = len(r) + } + r = append(r[:cursor-1], r[cursor:]...) + m.termPartial = string(r) + m.termCursor = cursor - 1 + m.termHistPos = -1 +} + +func (m *monitorModel) deleteConsoleAt() { + r := []rune(m.termPartial) + if len(r) == 0 { + return + } + cursor := m.termCursor + if cursor < 0 { + cursor = 0 + } + if cursor >= len(r) { + return + } + r = append(r[:cursor], r[cursor+1:]...) + m.termPartial = string(r) + m.termCursor = cursor + m.termHistPos = -1 +} + +func (m *monitorModel) deleteConsoleToStart() { + r := []rune(m.termPartial) + if len(r) == 0 || m.termCursor <= 0 { + return + } + cursor := m.termCursor + if cursor > len(r) { + cursor = len(r) + } + m.termPartial = string(r[cursor:]) + m.termCursor = 0 + m.termHistPos = -1 +} + +func (m *monitorModel) deleteConsoleToEnd() { + r := []rune(m.termPartial) + if len(r) == 0 { + return + } + cursor := m.termCursor + if cursor < 0 { + cursor = 0 + } + if cursor >= len(r) { + return + } + m.termPartial = string(r[:cursor]) + m.termHistPos = -1 +} + +func (m *monitorModel) deleteConsoleWord() { + r := []rune(m.termPartial) + if len(r) == 0 || m.termCursor <= 0 { + return + } + cursor := m.termCursor + if cursor > len(r) { + cursor = len(r) + } + i := cursor - 1 + for i >= 0 && (r[i] == ' ' || r[i] == '\t') { + i-- + } + for i >= 0 && r[i] != ' ' && r[i] != '\t' { + i-- + } + start := i + 1 + merged := append([]rune{}, r[:start]...) + merged = append(merged, r[cursor:]...) + m.termPartial = string(merged) + m.termCursor = start + m.termHistPos = -1 +} + +func highlightConsoleInput(line string) string { + line = strings.TrimRight(line, "\r\n") + if strings.TrimSpace(line) == "" { + return "" + } + isBuiltIn := func(s string) bool { + switch strings.ToLower(strings.TrimSpace(s)) { + case "help", "history", "clear", "overview", "clusters", "network", "settings", "quit", "exit", "console", "privacy": + return true + default: + return false + } + } + colorToken := func(tok string, pos int) string { + switch { + case pos == 0 && isBuiltIn(tok): + return okStyle.Render(tok) + case pos == 0: + return accentStyle.Bold(true).Render(tok) + case strings.HasPrefix(tok, "--"): + return accentStyle.Render(tok) + case strings.HasPrefix(tok, "-"): + return softStyle.Render(tok) + case len(tok) >= 2 && ((tok[0] == '"' && tok[len(tok)-1] == '"') || (tok[0] == '\'' && tok[len(tok)-1] == '\'')): + return warnStyle.Render(tok) + default: + return brightStyle.Render(tok) + } + } + + var out strings.Builder + tokenPos := 0 + for i := 0; i < len(line); { + r, size := utf8.DecodeRuneInString(line[i:]) + if r == utf8.RuneError && size == 1 { + out.WriteByte(line[i]) + i++ + continue + } + if r == ' ' || r == '\t' { + out.WriteRune(r) + i += size + continue + } + j := i + for j < len(line) { + rr, ss := utf8.DecodeRuneInString(line[j:]) + if rr == ' ' || rr == '\t' { + break + } + j += ss + } + tok := line[i:j] + out.WriteString(colorToken(tok, tokenPos)) + tokenPos++ + i = j + } + return out.String() +} + +func (m *monitorModel) consoleAutocomplete() { + r := []rune(m.termPartial) + cursor := m.termCursor + if cursor < 0 { + cursor = 0 + } + if cursor > len(r) { + cursor = len(r) + } + + left := r[:cursor] + right := r[cursor:] + + start := len(left) + for start > 0 { + ch := left[start-1] + if ch == ' ' || ch == '\t' { + break + } + start-- + } + + prefix := strings.ToLower(string(left[start:])) + base := strings.Fields(string(left[:start])) + cands := m.consoleCandidates(base, prefix) + if len(cands) == 0 { + return + } + + if len(cands) == 1 { + replacement := cands[0] + newLeft := string(left[:start]) + replacement + if len(right) == 0 || (len(right) > 0 && right[0] != ' ') { + newLeft += " " + } + m.termPartial = newLeft + string(right) + m.termCursor = len([]rune(newLeft)) + return + } + + common := commonPrefix(cands) + if len(common) > len(prefix) { + newLeft := string(left[:start]) + common + m.termPartial = newLeft + string(right) + m.termCursor = len([]rune(newLeft)) + return + } + + m.pushTerminalLine("suggestions: " + strings.Join(cands, " ")) +} + +func (m monitorModel) consoleCandidates(base []string, prefix string) []string { + top := []string{ + "cluster", "alerts", "alert", "software", "plugins", "bot", "ping", "connect", + "list", "show", "current", "use", "bootstrap", "stats", + "kvm", "lxc", "lxd", "bird", "frr", + "help", "history", "clear", "overview", "clusters", "network", "settings", "quit", "exit", + "console", + } + clusterSubs := []string{ + "connect", "add", "list", "ls", "show", "get", "current", "use", + "ping", "check", "bootstrap", "stats", "alert", "alerts", "software", "plugins", + "disconnect", "remove", "rm", "help", + } + alertSubs := []string{"show", "set"} + softwareSubs := []string{"show", "scan"} + kvmSubs := []string{"list", "start", "stop", "reboot", "restart", "destroy", "top", "net-top"} + lxcSubs := []string{"list", "start", "stop", "restart", "top", "net-top", "stats", "info", "show"} + birdSubs := []string{"status", "protocols", "routes"} + frrSubs := []string{"status", "routes", "bgp", "ospf"} + botSubs := []string{"telegram"} + botTelegramSubs := []string{"show", "set", "disable", "run"} + consoleSubs := []string{"full", "fullscreen", "dock", "toggle"} + + var pool []string + switch { + case len(base) == 0: + pool = top + case len(base) == 1 && strings.EqualFold(base[0], "cluster"): + pool = clusterSubs + case len(base) == 1 && wantsClusterName(base[0]): + pool = m.clusterNameCandidates() + case len(base) == 2 && strings.EqualFold(base[0], "cluster") && wantsClusterName(base[1]): + pool = m.clusterNameCandidates() + case len(base) == 1 && (strings.EqualFold(base[0], "alert") || strings.EqualFold(base[0], "alerts")): + pool = alertSubs + case len(base) == 2 && (strings.EqualFold(base[0], "alert") || strings.EqualFold(base[0], "alerts")): + if strings.EqualFold(base[1], "set") || strings.EqualFold(base[1], "show") { + pool = m.clusterNameCandidates() + } + case len(base) == 1 && (strings.EqualFold(base[0], "software") || strings.EqualFold(base[0], "plugins")): + pool = softwareSubs + case len(base) == 2 && strings.EqualFold(base[0], "cluster") && + (strings.EqualFold(base[1], "software") || strings.EqualFold(base[1], "plugins")): + pool = softwareSubs + case len(base) == 1 && strings.EqualFold(base[0], "kvm"): + pool = kvmSubs + case len(base) == 1 && (strings.EqualFold(base[0], "lxc") || strings.EqualFold(base[0], "lxd")): + pool = lxcSubs + case len(base) == 1 && strings.EqualFold(base[0], "bird"): + pool = birdSubs + case len(base) == 1 && strings.EqualFold(base[0], "frr"): + pool = frrSubs + case len(base) == 1 && strings.EqualFold(base[0], "bot"): + pool = botSubs + case len(base) == 2 && strings.EqualFold(base[0], "bot") && strings.EqualFold(base[1], "telegram"): + pool = botTelegramSubs + case len(base) == 1 && strings.EqualFold(base[0], "console"): + pool = consoleSubs + case len(base) == 2 && strings.EqualFold(base[0], "software") && + (strings.EqualFold(base[1], "scan") || strings.EqualFold(base[1], "show")): + pool = m.clusterNameCandidates() + case len(base) == 3 && (strings.EqualFold(base[0], "kvm") || strings.EqualFold(base[0], "lxc") || strings.EqualFold(base[0], "lxd") || + strings.EqualFold(base[0], "bird") || strings.EqualFold(base[0], "frr")) && + (strings.EqualFold(base[1], "list") || strings.EqualFold(base[1], "status") || + strings.EqualFold(base[1], "top") || strings.EqualFold(base[1], "net-top") || + strings.EqualFold(base[1], "stats") || strings.EqualFold(base[1], "info") || strings.EqualFold(base[1], "show") || + strings.EqualFold(base[1], "routes") || strings.EqualFold(base[1], "protocols") || + strings.EqualFold(base[1], "bgp") || strings.EqualFold(base[1], "ospf")): + pool = m.clusterNameCandidates() + } + + if len(pool) == 0 { + return nil + } + out := make([]string, 0, len(pool)) + for _, item := range pool { + if prefix == "" || strings.HasPrefix(strings.ToLower(item), prefix) { + out = append(out, item) + } + } + sort.Strings(out) + return dedupeStrings(out) +} + +func (m monitorModel) clusterNameCandidates() []string { + clusters, _, err := m.svc.List() + if err != nil { + return nil + } + names := make([]string, 0, len(clusters)) + for _, c := range clusters { + if strings.TrimSpace(c.Name) != "" { + names = append(names, c.Name) + } + } + sort.Strings(names) + return dedupeStrings(names) +} + +func wantsClusterName(token string) bool { + switch strings.ToLower(strings.TrimSpace(token)) { + case "use", "show", "get", "ping", "check", "bootstrap", "stats", "disconnect", "remove", "rm", "connect", "software", "plugins": + return true + default: + return false + } +} + +func commonPrefix(values []string) string { + if len(values) == 0 { + return "" + } + p := values[0] + for _, v := range values[1:] { + for !strings.HasPrefix(v, p) && p != "" { + p = p[:len(p)-1] + } + if p == "" { + return "" + } + } + return p +} + +func dedupeStrings(in []string) []string { + if len(in) == 0 { + return nil + } + out := make([]string, 0, len(in)) + var prev string + for i, item := range in { + if i == 0 || item != prev { + out = append(out, item) + } + prev = item + } + return out +} + +func (m monitorModel) metricTabs() string { + tabs := []graphMetric{metricRX, metricTX, metricTotal} + parts := make([]string, 0, len(tabs)) + for _, t := range tabs { + label := metricLabel(t) + if t == m.metric { + parts = append(parts, "["+label+"]") + } else { + parts = append(parts, " "+label+" ") + } + } + return "metric: " + strings.Join(parts, " ") +} + +func (m monitorModel) networkRows() []networkRow { + lQuery := strings.ToLower(strings.TrimSpace(m.searchTerm)) + + byIfaceStat := make(map[string]agent.NetworkStat, len(m.stats.Network)) + for _, n := range m.stats.Network { + byIfaceStat[n.Interface] = n + } + + all := map[string]struct{}{} + for iface := range m.runtime.rates { + all[iface] = struct{}{} + } + for iface := range m.history { + all[iface] = struct{}{} + } + for iface := range byIfaceStat { + all[iface] = struct{}{} + } + + rows := make([]networkRow, 0, len(all)) + for iface := range all { + if lQuery != "" && !strings.Contains(strings.ToLower(iface), lQuery) { + continue + } + + rate := m.runtime.rates[iface] + stat := byIfaceStat[iface] + row := networkRow{ + Interface: iface, + CurRxMbps: rate.RxMbps, + CurTxMbps: rate.TxMbps, + CurTotalMbps: rate.RxMbps + rate.TxMbps, + RxDrops: stat.RxDrops, + TxDrops: stat.TxDrops, + Samples: len(m.history[iface]), + Spark: strings.Repeat(".", 18), + } + rows = append(rows, row) + } + + sort.Slice(rows, func(i, j int) bool { + if rows[i].CurTotalMbps != rows[j].CurTotalMbps { + return rows[i].CurTotalMbps > rows[j].CurTotalMbps + } + if rows[i].Samples != rows[j].Samples { + return rows[i].Samples > rows[j].Samples + } + return rows[i].Interface < rows[j].Interface + }) + return rows +} + +func (m monitorModel) enrichNetworkRows(rows []networkRow) []networkRow { + out := append([]networkRow(nil), rows...) + for i := range out { + points := m.history[out[i].Interface] + if len(points) == 0 { + out[i].Spark = strings.Repeat(".", 18) + continue + } + seriesStats := computeSeriesStats(points) + out[i].AvgTotalMbps = seriesStats.AvgTotalMbps + out[i].PeakTotal = seriesStats.PeakTotalMbps + out[i].ConsumedByte = seriesStats.ConsumedBytes + out[i].Samples = seriesStats.Samples + out[i].Spark = sparklineFromHistory(points, 18) + } + return out +} + +func sparklineFromHistory(points []ifaceHistoryPoint, width int) string { + if len(points) == 0 { + return strings.Repeat(".", max(1, width)) + } + totalSeries := make([]float64, 0, len(points)) + for _, p := range points { + totalSeries = append(totalSeries, p.RxMbps+p.TxMbps) + } + return sparklineASCII(totalSeries, width) +} + +type seriesStats struct { + AvgTotalMbps float64 + PeakTotalMbps float64 + ConsumedBytes uint64 + Samples int +} + +type metricStats struct { + PeakMbps float64 + PeakDuration time.Duration + Samples int +} + +func computeSeriesStats(points []ifaceHistoryPoint) seriesStats { + if len(points) == 0 { + return seriesStats{} + } + + total := 0.0 + peak := 0.0 + for _, p := range points { + v := p.RxMbps + p.TxMbps + total += v + if v > peak { + peak = v + } + } + + consumed := integrateConsumedBytes(points, metricTotal) + return seriesStats{ + AvgTotalMbps: round2(total / float64(len(points))), + PeakTotalMbps: round2(peak), + ConsumedBytes: consumed, + Samples: len(points), + } +} + +func computeMetricStats(points []ifaceHistoryPoint, metric graphMetric) metricStats { + if len(points) == 0 { + return metricStats{} + } + + peak := 0.0 + for _, p := range points { + v := metricPointValue(p, metric) + if v > peak { + peak = v + } + } + + return metricStats{ + PeakMbps: round2(peak), + PeakDuration: estimatePeakHoldDuration(points, metric), + Samples: len(points), + } +} + +func integrateConsumedBytes(points []ifaceHistoryPoint, metric graphMetric) uint64 { + if len(points) < 2 { + return 0 + } + + // Assume points are already chronological. + totalBytes := 0.0 + for i := 1; i < len(points); i++ { + prev := points[i-1] + cur := points[i] + dt := cur.At.Sub(prev.At).Seconds() + if dt <= 0 || dt > 3600 { + continue + } + prevMbps := metricPointValue(prev, metric) + curMbps := metricPointValue(cur, metric) + avgMbps := (prevMbps + curMbps) / 2 + totalBytes += (avgMbps * 1_000_000.0 / 8.0) * dt + } + if totalBytes <= 0 { + return 0 + } + return uint64(totalBytes) +} + +func metricUsageBytesSince(points []ifaceHistoryPoint, metric graphMetric, since time.Time) uint64 { + if len(points) < 2 { + return 0 + } + if since.IsZero() { + return integrateConsumedBytes(points, metric) + } + + filtered := make([]ifaceHistoryPoint, 0, len(points)) + for _, p := range points { + if p.At.After(since) || p.At.Equal(since) { + filtered = append(filtered, p) + } + } + return integrateConsumedBytes(filtered, metric) +} + +func estimatePeakHoldDuration(points []ifaceHistoryPoint, metric graphMetric) time.Duration { + if len(points) < 2 { + return 0 + } + + peak := 0.0 + for _, p := range points { + v := metricPointValue(p, metric) + if v > peak { + peak = v + } + } + if peak <= 0 { + return 0 + } + + threshold := peak * 0.95 + var longest time.Duration + var run time.Duration + for i := 1; i < len(points); i++ { + prev := points[i-1] + cur := points[i] + dt := cur.At.Sub(prev.At) + if dt <= 0 || dt > time.Hour { + run = 0 + continue + } + v1 := metricPointValue(prev, metric) + v2 := metricPointValue(cur, metric) + if v1 >= threshold && v2 >= threshold { + run += dt + if run > longest { + longest = run + } + } else { + run = 0 + } + } + return longest +} + +func metricSeries(points []ifaceHistoryPoint, metric graphMetric) []float64 { + out := make([]float64, 0, len(points)) + for _, p := range points { + out = append(out, metricPointValue(p, metric)) + } + return out +} + +func metricCurrent(row networkRow, metric graphMetric) float64 { + switch metric { + case metricRX: + return row.CurRxMbps + case metricTX: + return row.CurTxMbps + default: + return row.CurTotalMbps + } +} + +func metricPointValue(p ifaceHistoryPoint, metric graphMetric) float64 { + switch metric { + case metricRX: + return p.RxMbps + case metricTX: + return p.TxMbps + default: + return p.RxMbps + p.TxMbps + } +} + +func metricLabel(metric graphMetric) string { + switch metric { + case metricRX: + return "RX" + case metricTX: + return "TX" + default: + return "TOTAL" + } +} + +func nextMetric(metric graphMetric) graphMetric { + switch metric { + case metricRX: + return metricTX + case metricTX: + return metricTotal + default: + return metricRX + } +} + +func prevMetric(metric graphMetric) graphMetric { + switch metric { + case metricRX: + return metricTotal + case metricTX: + return metricRX + default: + return metricTX + } +} + +func renderBigGraphASCII(values []float64, width, height int) string { + if width < 20 { + width = 20 + } + if height < 4 { + height = 4 + } + if len(values) == 0 { + return dimStyle.Render("no history points for graph") + } + + series := resample(values, width) + maxV := 0.0 + for _, v := range series { + if v > maxV { + maxV = v + } + } + if maxV <= 0 { + return dimStyle.Render(strings.Repeat(".", width)) + } + + var b strings.Builder + for row := height; row >= 1; row-- { + levelVal := maxV * float64(row) / float64(height) + b.WriteString(fmt.Sprintf("%7.2f |", levelVal)) + for _, v := range series { + if v >= levelVal { + b.WriteByte('#') + } else { + b.WriteByte(' ') + } + } + b.WriteByte('\n') + } + b.WriteString("--------+") + b.WriteString(strings.Repeat("-", len(series))) + b.WriteByte('\n') + b.WriteString(" 0") + return strings.TrimRight(b.String(), "\n") +} + +func paginationBounds(total, page, pageSize int) (start, end, totalPages int) { + if pageSize <= 0 { + pageSize = 8 + } + totalPages = pageCount(total, pageSize) + if totalPages < 1 { + totalPages = 1 + } + if page < 1 { + page = 1 + } + if page > totalPages { + page = totalPages + } + start = (page - 1) * pageSize + if start > total { + start = total + } + end = start + pageSize + if end > total { + end = total + } + return +} + +func pageCount(total, pageSize int) int { + if pageSize <= 0 { + pageSize = 8 + } + if total <= 0 { + return 1 + } + return (total + pageSize - 1) / pageSize +} + +func parseView(v string) monitorView { + switch strings.ToLower(strings.TrimSpace(v)) { + case "clusters", "cluster", "cluster-overview": + return viewClusters + case "network", "net": + return viewNetwork + case "settings", "cfg", "config": + return viewSettings + case "docs", "doc", "help", "manual": + return viewDocs + default: + return viewOverview + } +} + +func defaultHistoryRange() historyRange { + now := time.Now().UTC() + return historyRange{ + Label: "30d", + Since: now.Add(-30 * 24 * time.Hour), + All: false, + } +} + +func parseHistoryRange(token string) (historyRange, error) { + now := time.Now().UTC() + switch strings.ToLower(strings.TrimSpace(token)) { + case "1h": + return historyRange{Label: "1h", Since: now.Add(-time.Hour)}, nil + case "24h", "1d": + return historyRange{Label: "24h", Since: now.Add(-24 * time.Hour)}, nil + case "7d", "1w": + return historyRange{Label: "7d", Since: now.Add(-7 * 24 * time.Hour)}, nil + case "30d", "1m", "month": + return historyRange{Label: "30d", Since: now.Add(-30 * 24 * time.Hour)}, nil + case "all": + return historyRange{Label: "all", All: true}, nil + default: + return historyRange{}, errors.New("invalid range, allowed: 1h|24h|7d|30d|all") + } +} + +func sparklineASCII(values []float64, width int) string { + if width <= 0 { + width = 16 + } + if len(values) == 0 { + return strings.Repeat(".", width) + } + + sampled := resample(values, width) + maxV := 0.0 + for _, v := range sampled { + if v > maxV { + maxV = v + } + } + if maxV <= 0 { + return strings.Repeat(".", len(sampled)) + } + + levels := []byte{'.', ':', '-', '=', '+', '*', '#', '%', '@'} + var b strings.Builder + b.Grow(len(sampled)) + for _, v := range sampled { + ratio := v / maxV + if ratio < 0 { + ratio = 0 + } + if ratio > 1 { + ratio = 1 + } + idx := int(math.Round(ratio * float64(len(levels)-1))) + if idx < 0 { + idx = 0 + } + if idx >= len(levels) { + idx = len(levels) - 1 + } + b.WriteByte(levels[idx]) + } + return b.String() +} + +func resample(values []float64, width int) []float64 { + if width <= 0 { + return []float64{} + } + if len(values) <= width { + out := make([]float64, 0, width) + out = append(out, values...) + for len(out) < width { + out = append(out, values[len(values)-1]) + } + return out + } + + out := make([]float64, 0, width) + step := float64(len(values)) / float64(width) + for i := 0; i < width; i++ { + from := int(math.Floor(float64(i) * step)) + to := int(math.Floor(float64(i+1) * step)) + if to <= from { + to = from + 1 + } + if from < 0 { + from = 0 + } + if to > len(values) { + to = len(values) + } + sum := 0.0 + for j := from; j < to; j++ { + sum += values[j] + } + out = append(out, sum/float64(to-from)) + } + return out +} + +func humanBitsRate(bytes uint64) string { + if bytes == 0 { + return "0B" + } + return humanBytes(bytes) +} + +func renderPanel(title, body string, width int) string { + return renderPanelStyled(panelStyle, title, body, width) +} + +// clipBodyToHeight returns EXACTLY `height` lines from body starting at +// `offset`. Pads with empty lines if body is shorter, truncates with scroll +// indicators if longer. Returns the clamped offset and whether clipping +// occurred. A fixed line count per panel makes the overall View deterministic +// across frames, which is what keeps bubbletea's diff renderer stable. +func clipBodyToHeight(body string, height, offset int) (string, int, bool) { + if height <= 0 { + return "", 0, false + } + lines := strings.Split(body, "\n") + total := len(lines) + if total <= height { + out := make([]string, height) + copy(out, lines) + return strings.Join(out, "\n"), 0, false + } + maxOffset := total - height + if offset > maxOffset { + offset = maxOffset + } + if offset < 0 { + offset = 0 + } + window := make([]string, height) + copy(window, lines[offset:offset+height]) + if offset > 0 && height >= 1 { + window[0] = dimStyle.Render(fmt.Sprintf("↑ %d more (alt+↑/↓ to scroll)", offset)) + } + if offset < maxOffset && height >= 1 { + window[height-1] = dimStyle.Render(fmt.Sprintf("↓ %d more (alt+↑/↓ to scroll)", maxOffset-offset)) + } + return strings.Join(window, "\n"), offset, true +} + +// visibleHeight counts the number of newline-separated lines in s. +func visibleHeight(s string) int { + if s == "" { + return 0 + } + return strings.Count(s, "\n") + 1 +} + +func renderPanelGrid(style lipgloss.Style, title, body string, width, height int) string { + if width < 24 { + width = 24 + } + if height < 3 { + height = 3 + } + header := titleStyle.Render(title) + content := lipgloss.JoinVertical(lipgloss.Left, header, body) + return style.Copy(). + Width(width). + MaxWidth(width). + Height(height). + MaxHeight(height). + Render(content) +} + +func renderFixedPanel(style lipgloss.Style, title, body string, width, height, offset int) (string, int) { + if height < 5 { + height = 5 + } + bodyHeight := height - 3 // 2 borders + 1 title line + if bodyHeight < 1 { + bodyHeight = 1 + } + clipped, clampedOffset, _ := clipBodyToHeight(body, bodyHeight, offset) + return renderPanelGrid(style, title, clipped, width, height), clampedOffset +} + +func renderPanelStyled(style lipgloss.Style, title, body string, width int) string { + if width < 24 { + width = 24 + } + header := titleStyle.Render(title) + content := lipgloss.JoinVertical(lipgloss.Left, header, body) + return style.Width(width).Render(content) +} + +func barASCII(percent float64, width int) string { + if width < 4 { + width = 4 + } + if percent < 0 { + percent = 0 + } + if percent > 100 { + percent = 100 + } + + filled := int(math.Round((percent / 100) * float64(width))) + if filled < 0 { + filled = 0 + } + if filled > width { + filled = width + } + return "[" + strings.Repeat("=", filled) + strings.Repeat("-", width-filled) + "]" +} + +func truncate(s string, limit int) string { + if limit <= 0 { + return "" + } + if len(s) <= limit { + return s + } + if limit <= 3 { + return s[:limit] + } + return s[:limit-3] + "..." +} + +func truncateRunes(s string, limit int) string { + if limit <= 0 { + return "" + } + rs := []rune(s) + if len(rs) <= limit { + return s + } + if limit <= 3 { + return string(rs[:limit]) + } + return string(rs[:limit-1]) + "…" +} + +func visibleLen(s string) int { + stripped := ansiCSIRegex.ReplaceAllString(s, "") + stripped = ansiOSCRegex.ReplaceAllString(stripped, "") + return len([]rune(stripped)) +} + +func truncateVisible(s string, limit int) string { + if limit <= 0 { + return "" + } + if visibleLen(s) <= limit { + return s + } + runes := []rune(s) + var b strings.Builder + visible := 0 + target := limit - 1 + for i := 0; i < len(runes); { + r := runes[i] + if r == 0x1b && i+1 < len(runes) && runes[i+1] == '[' { + b.WriteRune(r) + i++ + b.WriteRune(runes[i]) + i++ + for i < len(runes) { + c := runes[i] + b.WriteRune(c) + i++ + if c >= 0x40 && c <= 0x7e { + break + } + } + continue + } + if visible >= target { + break + } + b.WriteRune(r) + visible++ + i++ + } + b.WriteString("\x1b[0m…") + return b.String() +} + +func truncateMultiline(s string, limit int) string { + if limit <= 0 { + return "" + } + lines := strings.Split(s, "\n") + for i := range lines { + lines[i] = truncate(lines[i], limit) + } + return strings.Join(lines, "\n") +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func fetchAgentStats(svc *cluster.Service, selector string) (agent.StatsResponse, error) { + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + return svc.AgentStatsTyped(ctx, selector) +} + +func (m *monitorRuntime) update(stats agent.StatsResponse) { + now := stats.Timestamp + if now.IsZero() { + now = time.Now().UTC() + } + + curr := map[string]agent.NetworkStat{} + for _, n := range stats.Network { + curr[n.Interface] = n + } + + if !m.prevAt.IsZero() { + dt := now.Sub(m.prevAt).Seconds() + if dt > 0 { + for iface, cur := range curr { + if prev, ok := m.prevNet[iface]; ok { + rxDelta := diffCounter(cur.RxBytes, prev.RxBytes) + txDelta := diffCounter(cur.TxBytes, prev.TxBytes) + m.rates[iface] = ifaceRate{ + RxMbps: round2((float64(rxDelta) * 8 / 1_000_000) / dt), + TxMbps: round2((float64(txDelta) * 8 / 1_000_000) / dt), + } + } + } + } + } + + m.prevAt = now + m.prevNet = curr + + ifaces := interfaceNames(stats.Network) + if len(ifaces) == 0 { + m.selectedIface = "" + return + } + if m.selectedIface == "" { + m.selectedIface = ifaces[0] + return + } + for _, iface := range ifaces { + if iface == m.selectedIface { + return + } + } + m.selectedIface = ifaces[0] +} + +func printStatsSnapshot(w io.Writer, stats agent.StatsResponse) { + pctColor := func(p float64) string { + s := fmt.Sprintf("%.1f%%", p) + switch { + case p >= 90: + return colorCrit(s) + case p >= 70: + return colorWarn(s) + default: + return colorOK(s) + } + } + + fmt.Fprintf(w, "%s %s\n", colorLabel("Time:"), colorDim(stats.Timestamp.Format(time.RFC3339))) + fmt.Fprintf(w, "%s %s %s\n", + colorLabel("Host:"), + colorAccent(stats.Host.Hostname), + colorMuted(fmt.Sprintf("(%s/%s)", stats.Host.OS, stats.Host.Arch)), + ) + fmt.Fprintf(w, "%s %s %s %s\n", + colorLabel("CPU: "), + pctColor(stats.CPU.UsagePercent), + colorMuted("| load"), + colorInfo(fmt.Sprintf("%.2f %.2f %.2f", stats.CPU.Load1, stats.CPU.Load5, stats.CPU.Load15)), + ) + fmt.Fprintf(w, "%s %s %s %s %s %s\n", + colorLabel("RAM: "), + pctColor(stats.Memory.UsedPercent), + colorMuted("|"), + colorValue(humanBytes(stats.Memory.UsedBytes)), + colorMuted("/"), + colorBlue(humanBytes(stats.Memory.TotalBytes)), + ) + if stats.Memory.SwapTotalBytes > 0 { + fmt.Fprintf(w, "%s %s %s %s %s %s\n", + colorLabel("Swap:"), + pctColor(stats.Memory.SwapUsedPct), + colorMuted("|"), + colorValue(humanBytes(stats.Memory.SwapUsedBytes)), + colorMuted("/"), + colorBlue(humanBytes(stats.Memory.SwapTotalBytes)), + ) + } + + tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", + colorHeader("DISK"), + colorHeader("USE"), + colorHeader("HEALTH"), + colorHeader("WARNINGS"), + ) + for _, d := range stats.Disk { + warn := strings.Join(d.Warnings, "; ") + warnColored := colorDim("-") + if warn != "" { + warnColored = colorWarn(warn) + } + health := emptyFallback(d.Health, "ok") + healthColored := colorOK(health) + switch strings.ToLower(health) { + case "critical", "fail", "failed": + healthColored = colorCrit(health) + case "warn", "warning", "degraded": + healthColored = colorWarn(health) + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\n", + colorInfo(d.MountPoint), + pctColor(d.UsedPercent), + healthColored, + warnColored, + ) + } + _ = tw.Flush() +} + +func collectAlerts(stats agent.StatsResponse, m *monitorRuntime) []string { + alerts := []string{} + p := m.policy + + cpuPct := stats.CPU.UsagePercent + if cpuPct <= 0 && stats.CPU.LogicalCores > 0 { + cpuPct = round2(math.Min((stats.CPU.Load1/float64(stats.CPU.LogicalCores))*100, 100)) + } + if cpuPct >= p.CPUWarnPercent { + alerts = append(alerts, fmt.Sprintf("CPU high: %.1f%% >= %.1f%%", cpuPct, p.CPUWarnPercent)) + } + if stats.Memory.UsedPercent >= p.RAMWarnPercent { + alerts = append(alerts, fmt.Sprintf("RAM high: %.1f%% >= %.1f%%", stats.Memory.UsedPercent, p.RAMWarnPercent)) + } + if stats.Memory.SwapTotalBytes > 0 && stats.Memory.SwapUsedPct >= p.SwapWarnPercent { + alerts = append(alerts, fmt.Sprintf("Swap high: %.1f%% >= %.1f%%", stats.Memory.SwapUsedPct, p.SwapWarnPercent)) + } + + for _, d := range stats.Disk { + if d.UsedPercent >= p.DiskWarnPercent { + alerts = append(alerts, fmt.Sprintf("Disk high on %s: %.1f%% >= %.1f%%", d.MountPoint, d.UsedPercent, p.DiskWarnPercent)) + } + if strings.ToLower(d.Health) == "critical" { + alerts = append(alerts, fmt.Sprintf("Disk critical on %s (%s)", d.MountPoint, strings.Join(d.Warnings, "; "))) + } else if len(d.Warnings) > 0 { + alerts = append(alerts, fmt.Sprintf("Disk warning on %s (%s)", d.MountPoint, strings.Join(d.Warnings, "; "))) + } + } + + rate := m.rates[m.selectedIface] + if p.NetWarnMbps > 0 { + if rate.RxMbps >= p.NetWarnMbps { + alerts = append(alerts, fmt.Sprintf("Network RX high on %s: %.1f Mbps >= %.1f Mbps", m.selectedIface, rate.RxMbps, p.NetWarnMbps)) + } + if rate.TxMbps >= p.NetWarnMbps { + alerts = append(alerts, fmt.Sprintf("Network TX high on %s: %.1f Mbps >= %.1f Mbps", m.selectedIface, rate.TxMbps, p.NetWarnMbps)) + } + } + if p.NetSustainEnabled && p.NetSustainMbps > 0 { + matched := 0 + for iface, r := range m.rates { + iface = strings.TrimSpace(iface) + if iface == "" { + continue + } + pinned := strings.TrimSpace(p.NetSustainIface) + if pinned != "" && !strings.EqualFold(iface, pinned) { + continue + } + if pinned == "" && !ifaceAllowedByFilters(iface, p.NetSustainInclude, p.NetSustainExclude) { + continue + } + if math.Max(r.RxMbps, r.TxMbps) >= p.NetSustainMbps { + matched++ + if matched <= 3 { + alerts = append(alerts, fmt.Sprintf("CRITICAL net candidate on %s: now %.1f/%.1f Mbps, sustained threshold %.1f Mbps for %d min", + iface, r.RxMbps, r.TxMbps, p.NetSustainMbps, p.NetSustainMinutes)) + } + } + } + if matched > 3 { + alerts = append(alerts, fmt.Sprintf("... plus %d more interfaces above sustained threshold now", matched-3)) + } + } + for _, w := range m.vmWarnings { + if strings.TrimSpace(w) == "" { + continue + } + alerts = append(alerts, "VM alert: "+w) + } + + return alerts +} + +func worstDisk(disks []agent.DiskStats) *agent.DiskStats { + if len(disks) == 0 { + return nil + } + worst := disks[0] + for _, d := range disks[1:] { + if d.UsedPercent > worst.UsedPercent { + worst = d + } + if strings.ToLower(d.Health) == "critical" && strings.ToLower(worst.Health) != "critical" { + worst = d + } + } + return &worst +} + +func humanBytes(v uint64) string { + const unit = 1024 + if v < unit { + return fmt.Sprintf("%d B", v) + } + div, exp := uint64(unit), 0 + for n := v / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + pre := "KMGTPE" + return fmt.Sprintf("%.2f %ciB", float64(v)/float64(div), pre[exp]) +} + +func interfaceNames(in []agent.NetworkStat) []string { + names := make([]string, 0, len(in)) + for _, n := range in { + names = append(names, n.Interface) + } + sort.Strings(names) + return names +} + +func cycleIface(names []string, current string, dir int) string { + if len(names) == 0 { + return "" + } + idx := 0 + for i, n := range names { + if n == current { + idx = i + break + } + } + idx += dir + if idx < 0 { + idx = len(names) - 1 + } + if idx >= len(names) { + idx = 0 + } + return names[idx] +} + +func diffCounter(cur, prev uint64) uint64 { + if cur >= prev { + return cur - prev + } + return 0 +} + +func errorsIsNoActive(err error) bool { + return errors.Is(err, cluster.ErrNoActiveCluster) +} + +func round2(v float64) float64 { + return math.Round(v*100) / 100 +} + +func (m monitorModel) currentCPUPct() float64 { + cpuPct := m.stats.CPU.UsagePercent + if cpuPct <= 0 && m.stats.CPU.LogicalCores > 0 { + cpuPct = round2(math.Min((m.stats.CPU.Load1/float64(m.stats.CPU.LogicalCores))*100, 100)) + } + return cpuPct +} diff --git a/internal/cli/monitor_live.go b/internal/cli/monitor_live.go new file mode 100644 index 0000000..6513c1e --- /dev/null +++ b/internal/cli/monitor_live.go @@ -0,0 +1,497 @@ +package cli + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +// liveClusterStat is one row on the live dashboard. Fields are best-effort; +// if the agent is unreachable we still render the row with an error so the +// operator sees the node exists but is degraded. +type liveClusterStat struct { + ClusterID string + Name string + Host string + Reachable bool + Err string + CPUPercent float64 + MemPercent float64 + MemUsed uint64 + MemTotal uint64 + DiskUsePct float64 + NetMbps float64 // current: last point from history + P95Mbps float64 // primary iface, 24h window + Uplink string + HostName string +} + +type monitorLiveMsg struct { + entries []liveClusterStat + err error +} + +type liveAutoTickMsg struct{} + +func liveAutoTickCmd() tea.Cmd { + return tea.Tick(liveAutoTickInterval, func(_ time.Time) tea.Msg { + return liveAutoTickMsg{} + }) +} + +const ( + livePageSize = 14 + liveFetchWorkers = 16 + liveAutoTickInterval = 5 * time.Second + liveP95HistoryWindow = 24 * time.Hour + liveFetchClusterTO = 5 * time.Second + liveFetchOverallTO = 25 * time.Second +) + +// fetchLiveCmd gathers live stats for every cluster with an installed agent +// in parallel, pairs each with P95 from history, and emits a sorted slice. +func (m monitorModel) fetchLiveCmd() tea.Cmd { + svc := m.svc + return func() tea.Msg { + clusters, _, err := svc.List() + if err != nil { + return monitorLiveMsg{err: err} + } + + ctx, cancel := context.WithTimeout(context.Background(), liveFetchOverallTO) + defer cancel() + + type work struct { + idx int + c cluster.Cluster + } + ch := make(chan work, len(clusters)) + results := make([]liveClusterStat, len(clusters)) + + var wg sync.WaitGroup + workers := liveFetchWorkers + if workers > len(clusters) { + workers = len(clusters) + } + if workers < 1 { + workers = 1 + } + + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for item := range ch { + results[item.idx] = collectLiveRow(ctx, svc, item.c) + } + }() + } + for i, c := range clusters { + ch <- work{idx: i, c: c} + } + close(ch) + wg.Wait() + + out := make([]liveClusterStat, 0, len(results)) + for _, r := range results { + if r.ClusterID == "" { + continue + } + out = append(out, r) + } + return monitorLiveMsg{entries: out} + } +} + +func collectLiveRow(ctx context.Context, svc *cluster.Service, c cluster.Cluster) liveClusterStat { + row := liveClusterStat{ + ClusterID: c.ID, + Name: c.Name, + Host: c.Host, + } + if !c.Agent.Installed { + row.Err = "agent not installed" + return row + } + callCtx, cancel := context.WithTimeout(ctx, liveFetchClusterTO) + defer cancel() + stats, err := svc.AgentStatsTyped(callCtx, c.ID) + if err != nil { + row.Err = err.Error() + return row + } + row.Reachable = true + row.CPUPercent = stats.CPU.UsagePercent + row.MemPercent = stats.Memory.UsedPercent + row.MemUsed = stats.Memory.UsedBytes + row.MemTotal = stats.Memory.TotalBytes + row.HostName = stats.Host.Hostname + var maxDisk float64 + for _, d := range stats.Disk { + if d.UsedPercent > maxDisk { + maxDisk = d.UsedPercent + } + } + row.DiskUsePct = maxDisk + + // Historical P95 from the persisted store (populated by the sampler + // running inside TUI or bot daemon). + if store := svc.NetworkStore(); store != nil { + since := time.Now().Add(-liveP95HistoryWindow) + if snaps, err := store.Load(c.ID, since); err == nil && len(snaps) > 0 { + primary := history.PrimaryInterface(snaps) + row.Uplink = primary + series := history.AggregateNodeSeries(snaps, primary) + if len(series) > 0 { + row.P95Mbps = history.PercentileMbps(series, 95) + row.NetMbps = series[len(series)-1].TotalMbps + } + } + } + + // Fallback / live reading: take a second stats snapshot after a short + // pause and derive the instantaneous per-interface Mbps ourselves. + // This makes NET Mbps meaningful even when the history store is empty + // (e.g. the sampler hasn't been running long). + time.Sleep(700 * time.Millisecond) + callCtx2, cancel2 := context.WithTimeout(ctx, liveFetchClusterTO) + defer cancel2() + stats2, err := svc.AgentStatsTyped(callCtx2, c.ID) + if err != nil { + return row + } + elapsed := stats2.Timestamp.Sub(stats.Timestamp).Seconds() + if elapsed <= 0 { + elapsed = 0.7 + } + prev := make(map[string]struct { + rx, tx uint64 + }, len(stats.Network)) + for _, n := range stats.Network { + prev[n.Interface] = struct{ rx, tx uint64 }{n.RxBytes, n.TxBytes} + } + bestName := "" + bestMbps := -1.0 + for _, n := range stats2.Network { + if isLiveVirtual(n.Interface) { + continue + } + p, ok := prev[n.Interface] + if !ok { + continue + } + var rxB, txB uint64 + if n.RxBytes >= p.rx { + rxB = n.RxBytes - p.rx + } + if n.TxBytes >= p.tx { + txB = n.TxBytes - p.tx + } + rxMbps := float64(rxB) * 8 / elapsed / 1_000_000 + txMbps := float64(txB) * 8 / elapsed / 1_000_000 + m := rxMbps + if txMbps > m { + m = txMbps + } + if m > bestMbps { + bestMbps = m + bestName = n.Interface + } + } + if bestMbps >= 0 { + row.NetMbps = bestMbps + if row.Uplink == "" { + row.Uplink = bestName + } + } + return row +} + +// isLiveVirtual mirrors history.isVirtualIface but is local so we don't +// export the original. +func isLiveVirtual(name string) bool { + n := strings.ToLower(name) + if n == "" || n == "lo" { + return true + } + prefixes := []string{ + "lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr", + "cni", "flannel", "wg", "tun", "tailscale", "zt", "kube", + "cilium", "ovs", "podman", "dummy", + } + for _, p := range prefixes { + if strings.HasPrefix(n, p) { + return true + } + } + return false +} + +// handleLiveKey routes keys while the live dashboard is visible. +func (m monitorModel) handleLiveKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "o", "esc": + m.view = viewOverview + return m, nil + case "c": + m.view = viewClusters + return m, nil + case "r", "R": + m.liveLoading = true + m.setStatus("live: refreshing") + return m, m.fetchLiveCmd() + case "up", "k": + if m.liveCursor > 0 { + m.liveCursor-- + } + return m, nil + case "down", "j": + if m.liveCursor < len(m.liveEntries)-1 { + m.liveCursor++ + } + return m, nil + case "pgup": + if m.livePage > 0 { + m.livePage-- + } + return m, nil + case "pgdown": + pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize + if m.livePage < pages-1 { + m.livePage++ + } + return m, nil + case "home": + m.liveCursor = 0 + m.livePage = 0 + return m, nil + case "end": + m.liveCursor = len(m.liveEntries) - 1 + if m.liveCursor < 0 { + m.liveCursor = 0 + } + pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize + if pages > 0 { + m.livePage = pages - 1 + } + return m, nil + case "enter": + if m.livePinned == nil { + m.livePinned = map[string]bool{} + } + rows := m.sortedLiveEntries() + if m.liveCursor >= 0 && m.liveCursor < len(rows) { + id := rows[m.liveCursor].ClusterID + if m.livePinned[id] { + delete(m.livePinned, id) + m.setStatus("live: unpinned " + rows[m.liveCursor].Name) + } else { + m.livePinned[id] = true + m.setStatus("live: pinned " + rows[m.liveCursor].Name) + } + } + return m, nil + case "1": + m.liveSort = 0 + return m, nil + case "2": + m.liveSort = 1 + return m, nil + case "3": + m.liveSort = 2 + return m, nil + case "4": + m.liveSort = 3 + return m, nil + case "5": + m.liveSort = 4 + return m, nil + } + return m, nil +} + +// sortedLiveEntries returns entries sorted by the active metric. Pinned +// clusters always float to the top so the operator keeps eyes on them even +// when their usage changes. +func (m monitorModel) sortedLiveEntries() []liveClusterStat { + entries := make([]liveClusterStat, len(m.liveEntries)) + copy(entries, m.liveEntries) + key := func(r liveClusterStat) float64 { + switch m.liveSort { + case 1: + return r.MemPercent + case 2: + return r.NetMbps + case 3: + return r.P95Mbps + case 4: + return r.DiskUsePct + default: + return r.CPUPercent + } + } + sort.SliceStable(entries, func(i, j int) bool { + pi := m.livePinned[entries[i].ClusterID] + pj := m.livePinned[entries[j].ClusterID] + if pi != pj { + return pi + } + return key(entries[i]) > key(entries[j]) + }) + return entries +} + +func liveSortLabel(n int) string { + switch n { + case 1: + return "ram" + case 2: + return "net" + case 3: + return "p95" + case 4: + return "disk" + default: + return "cpu" + } +} + +// renderLiveDashboard paints the paginated live view. +func (m monitorModel) renderLiveDashboard(width int) string { + title := titleStyle.Render("━━ Live cluster fleet ━━") + help := dimStyle.Render("sort: 1=cpu 2=ram 3=net 4=p95 5=disk enter pin r refresh ↑/↓ select pgup/pgdn page o back") + + if m.liveLoading && len(m.liveEntries) == 0 { + body := dimStyle.Render("collecting stats from every cluster…") + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) + } + if m.liveError != "" && len(m.liveEntries) == 0 { + body := critStyle.Render("error: " + m.liveError) + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) + } + if len(m.liveEntries) == 0 { + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", + dimStyle.Render("no clusters with installed agents")) + } + + entries := m.sortedLiveEntries() + total := len(entries) + pages := (total + livePageSize - 1) / livePageSize + if pages == 0 { + pages = 1 + } + if m.livePage >= pages { + m.livePage = pages - 1 + } + start := m.livePage * livePageSize + end := start + livePageSize + if end > total { + end = total + } + page := entries[start:end] + + headerLine := fmt.Sprintf(" %-3s %-16s %-3s %6s %6s %14s %12s %12s %6s %-10s", + "#", "NAME", "PIN", "CPU%", "RAM%", "RAM", "NET", "P95", "DISK%", "UPLINK") + lines := []string{title, help, "", + dimStyle.Render(fmt.Sprintf("sorted by %s · page %d/%d · %d clusters · refresh every %s · enter=pin", + liveSortLabel(m.liveSort), m.livePage+1, pages, total, liveAutoTickInterval)), + "", + accentStyle.Bold(true).Render(headerLine), + } + + for i, row := range page { + absoluteIdx := start + i + selected := absoluteIdx == m.liveCursor + pin := " " + if m.livePinned[row.ClusterID] { + pin = "⚑" + } + name := row.Name + if len(name) > 16 { + name = name[:13] + "..." + } + if !row.Reachable { + line := fmt.Sprintf(" %-3d %-16s %-3s %s", + absoluteIdx+1, name, pin, critStyle.Render("unreachable: "+clipRight(row.Err, 60))) + if selected { + line = selectedRowStyle.Render(line) + } + lines = append(lines, line) + continue + } + + cpuStr := colorByThreshold(fmt.Sprintf("%5.1f", row.CPUPercent), row.CPUPercent, 50, 80) + ramStr := colorByThreshold(fmt.Sprintf("%5.1f", row.MemPercent), row.MemPercent, 50, 80) + ramAbs := fmt.Sprintf("%7s/%-6s", humanBytesUint(row.MemUsed), humanBytesUint(row.MemTotal)) + netStr := fmt.Sprintf("%12s", formatMbpsHuman(row.NetMbps)) + var p95Str string + if row.P95Mbps > 0 { + p95Str = fmt.Sprintf("%12s", formatMbpsHuman(row.P95Mbps)) + } else { + p95Str = dimStyle.Render(fmt.Sprintf("%12s", "— (no hist)")) + } + diskStr := colorByThreshold(fmt.Sprintf("%5.1f", row.DiskUsePct), row.DiskUsePct, 70, 90) + uplink := row.Uplink + if uplink == "" { + uplink = "-" + } + if len(uplink) > 10 { + uplink = uplink[:10] + } + + body := fmt.Sprintf(" %-3d %-16s %-3s %s %s %14s %s %s %s %-10s", + absoluteIdx+1, name, pin, + cpuStr, ramStr, ramAbs, netStr, p95Str, diskStr, uplink) + if selected { + body = selectedRowStyle.Render(body) + } + lines = append(lines, body) + } + + // Stats footer. + var totalCPU, totalMem, totalNet, totalP95 float64 + reachable := 0 + for _, r := range entries { + if !r.Reachable { + continue + } + reachable++ + totalCPU += r.CPUPercent + totalMem += r.MemPercent + totalNet += r.NetMbps + totalP95 += r.P95Mbps + } + if reachable > 0 { + lines = append(lines, "", + dimStyle.Render(fmt.Sprintf("fleet avg: cpu %.1f%% · ram %.1f%% · net %s · p95 %s · reachable %d/%d", + totalCPU/float64(reachable), + totalMem/float64(reachable), + formatMbpsHuman(totalNet), + formatMbpsHuman(totalP95), + reachable, total))) + } + + return strings.Join(lines, "\n") +} + +var selectedRowStyle = lipgloss.NewStyle().Background(lipgloss.Color("#3A3A3A")).Bold(true) + +func colorByThreshold(text string, value, warn, crit float64) string { + switch { + case value >= crit: + return critStyle.Render(text) + case value >= warn: + return warnStyle.Render(text) + default: + return okStyle.Render(text) + } +} diff --git a/internal/cli/monitor_livecmd.go b/internal/cli/monitor_livecmd.go new file mode 100644 index 0000000..d41aa09 --- /dev/null +++ b/internal/cli/monitor_livecmd.go @@ -0,0 +1,306 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" +) + +// livePluginSpec describes an invocation to rerun on a ticker. +type livePluginSpec struct { + Tool string + Action string + Selector string + Params []string + Interval time.Duration +} + +// Display returns a short human label for the header. +func (s livePluginSpec) Display() string { + parts := []string{s.Tool} + if s.Action != "" { + parts = append(parts, s.Action) + } + parts = append(parts, s.Params...) + out := strings.Join(parts, " ") + if s.Selector != "" { + out += " (@" + s.Selector + ")" + } + return out +} + +// detectLivePluginInvocation returns a filled livePluginSpec if the parsed +// argv is a plugin command (`kvm`, `lxc`, `lxd`, `bird`, `frr`) with a +// `--live` flag. The flag is consumed so downstream execution sees a clean +// argv without it. Optional `--interval=Ns` adjusts the refresh cadence. +func detectLivePluginInvocation(args []string) (livePluginSpec, bool) { + if len(args) == 0 { + return livePluginSpec{}, false + } + tool := strings.ToLower(strings.TrimSpace(args[0])) + switch tool { + case "kvm", "lxc", "lxd", "bird", "frr": + default: + return livePluginSpec{}, false + } + + hasLive := false + interval := 3 * time.Second + rest := args[1:] + clean := make([]string, 0, len(rest)) + for i := 0; i < len(rest); i++ { + a := strings.TrimSpace(rest[i]) + switch { + case a == "--live", a == "-L": + hasLive = true + case a == "--interval": + if i+1 < len(rest) { + if d, err := time.ParseDuration(rest[i+1]); err == nil { + interval = d + } + i++ + } + case strings.HasPrefix(a, "--interval="): + if d, err := time.ParseDuration(strings.TrimPrefix(a, "--interval=")); err == nil { + interval = d + } + default: + clean = append(clean, rest[i]) + } + } + if !hasLive { + return livePluginSpec{}, false + } + + // Resolve the cluster selector (--cluster/-c NAME) and pull it out of + // the remaining args so action+params stay clean. + selector, cleaned2, err := parseClusterSelectorArg(clean) + if err != nil { + return livePluginSpec{}, false + } + + action := "" + params := []string{} + if len(cleaned2) > 0 { + action = strings.ToLower(strings.TrimSpace(cleaned2[0])) + params = cleaned2[1:] + } + if action == "" { + switch tool { + case "kvm", "lxc", "lxd": + action = "list" + default: + action = "status" + } + } + if tool == "kvm" && action == "top" { + // kvm top is now an allocation/specs view, not a live telemetry stream. + return livePluginSpec{}, false + } + if interval < 500*time.Millisecond { + interval = 500 * time.Millisecond + } + return livePluginSpec{ + Tool: tool, + Action: action, + Selector: selector, + Params: params, + Interval: interval, + }, true +} + +// startLiveCmd flips the model into fullscreen live-command mode and +// schedules the first execution immediately. +func (m monitorModel) startLiveCmd(spec livePluginSpec) (tea.Model, tea.Cmd) { + m.liveCmdActive = true + m.liveCmdSpec = spec + m.liveCmdBuffer = "" + m.liveCmdErr = "" + m.liveCmdRunning = true + m.liveCmdInterval = spec.Interval + m.liveCmdScroll = 0 + // Leave the pxmon console so the live view owns the screen. + m.termMode = false + m.termFull = false + m.setStatus("live: " + spec.Display()) + return m, m.runLiveCmdCmd(spec) +} + +// liveCmdResultMsg carries one iteration of a live-command invocation. +type liveCmdResultMsg struct { + Spec livePluginSpec + Output string + Err string +} + +type liveCmdTickMsg struct{} + +// runLiveCmdCmd kicks off one execution of the plugin command in a +// goroutine. The result is delivered as liveCmdResultMsg and the caller +// schedules the next tick once it lands. +func (m monitorModel) runLiveCmdCmd(spec livePluginSpec) tea.Cmd { + svc := m.svc + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), pluginActionTimeout(spec.Tool, spec.Action, true)) + defer cancel() + out, err := svc.RunPluginAction(ctx, spec.Selector, spec.Tool, spec.Action, spec.Params) + res := liveCmdResultMsg{Spec: spec, Output: stripANSI(out)} + if err != nil { + res.Err = err.Error() + } + return res + } +} + +func liveCmdTickCmd(d time.Duration) tea.Cmd { + if d <= 0 { + d = 3 * time.Second + } + return tea.Tick(d, func(_ time.Time) tea.Msg { + return liveCmdTickMsg{} + }) +} + +// handleLiveCmdKey routes keys while the live-command fullscreen is active. +func (m monitorModel) handleLiveCmdKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "q", "esc", "ctrl+g", "ctrl+c": + m.liveCmdActive = false + m.liveCmdBuffer = "" + m.liveCmdRunning = false + m.setStatus("live: stopped") + return m, nil + case "r", "R": + m.liveCmdRunning = true + return m, m.runLiveCmdCmd(m.liveCmdSpec) + case "+": + if m.liveCmdInterval > time.Second { + m.liveCmdInterval -= time.Second + } + m.liveCmdSpec.Interval = m.liveCmdInterval + return m, nil + case "-": + m.liveCmdInterval += time.Second + if m.liveCmdInterval > 60*time.Second { + m.liveCmdInterval = 60 * time.Second + } + m.liveCmdSpec.Interval = m.liveCmdInterval + return m, nil + case "alt+up", "up", "k": + m.liveCmdScroll += 3 + return m, nil + case "alt+down", "down", "j": + m.liveCmdScroll -= 3 + if m.liveCmdScroll < 0 { + m.liveCmdScroll = 0 + } + return m, nil + case "alt+shift+up": + m.liveCmdScroll += 15 + return m, nil + case "alt+shift+down": + m.liveCmdScroll -= 15 + if m.liveCmdScroll < 0 { + m.liveCmdScroll = 0 + } + return m, nil + case "pgup": + m.liveCmdScroll += 20 + return m, nil + case "pgdown": + m.liveCmdScroll -= 20 + if m.liveCmdScroll < 0 { + m.liveCmdScroll = 0 + } + return m, nil + case "home": + return m, nil + case "end": + m.liveCmdScroll = 0 + return m, nil + case "alt+p": + m.privacyMode = !m.privacyMode + return m, nil + } + return m, nil +} + +// renderLiveCmdView paints the fullscreen live command view. +func (m monitorModel) renderLiveCmdView(width, height int) string { + spec := m.liveCmdSpec + title := accentStyle.Bold(true).Render(fmt.Sprintf(" live › %s ", spec.Display())) + var ageText string + if !m.liveCmdLastRun.IsZero() { + ageText = fmt.Sprintf("updated %s ago", time.Since(m.liveCmdLastRun).Round(time.Millisecond)) + } else { + ageText = "pending…" + } + state := "idle" + if m.liveCmdRunning { + state = "running" + } + meta := dimStyle.Render(fmt.Sprintf( + "interval %s · %s · %s", + m.liveCmdInterval.Round(time.Second), state, ageText, + )) + help := dimStyle.Render("q/esc exit · r refresh · +/- interval · alt+↑/↓ scroll · alt+shift+↑/↓ fast · alt+p privacy") + + bodyLines := strings.Split(m.liveCmdBuffer, "\n") + if m.liveCmdErr != "" { + bodyLines = append([]string{critStyle.Render("error: " + m.liveCmdErr), ""}, bodyLines...) + } + if len(bodyLines) == 0 || (len(bodyLines) == 1 && bodyLines[0] == "") { + bodyLines = []string{dimStyle.Render("(no output yet)")} + } + + viewportH := height - 6 + if viewportH < 5 { + viewportH = 5 + } + maxOffset := len(bodyLines) - viewportH + if maxOffset < 0 { + maxOffset = 0 + } + if m.liveCmdScroll > maxOffset { + m.liveCmdScroll = maxOffset + } + end := len(bodyLines) - m.liveCmdScroll + start := end - viewportH + if start < 0 { + start = 0 + } + if end > len(bodyLines) { + end = len(bodyLines) + } + windowed := bodyLines[start:end] + panelW := max(24, width) + bodyW := max(16, panelW-6) + for i := range windowed { + windowed[i] = truncateVisible(windowed[i], bodyW) + } + + scrollBadge := "" + if m.liveCmdScroll > 0 { + scrollBadge = warnStyle.Render(fmt.Sprintf(" ↑ scrolled +%d (end to follow) ", m.liveCmdScroll)) + } + + panel := lipgloss.NewStyle(). + BorderStyle(thinBorder). + BorderForeground(ccAccent). + Padding(0, 1). + Width(panelW). + MaxWidth(panelW). + Render(strings.Join(windowed, "\n")) + + headerLine := title + " " + meta + if scrollBadge != "" { + headerLine += " " + scrollBadge + } + headerLine = truncateVisible(headerLine, panelW) + help = truncateVisible(help, panelW) + return lipgloss.JoinVertical(lipgloss.Left, headerLine, help, panel) +} diff --git a/internal/cli/monitor_privacy.go b/internal/cli/monitor_privacy.go new file mode 100644 index 0000000..02c2f25 --- /dev/null +++ b/internal/cli/monitor_privacy.go @@ -0,0 +1,85 @@ +package cli + +import ( + "regexp" + "strings" +) + +// Privacy mode redacts sensitive tokens (IPs, long secret-like strings, +// hostnames, MAC addresses, ssh key material) from rendered output. The +// replacement uses a shifted block pattern (▚▞) which preserves token +// length so the layout doesn't shift but makes the content obviously +// unreadable — think "frosted glass" rather than the usual `****`. + +var ( + privacyIPv4 = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`) + privacyIPv6 = regexp.MustCompile(`\b(?:[0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]{0,}\b`) + privacyMAC = regexp.MustCompile(`\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\b`) + privacyHost = regexp.MustCompile(`\b[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}(?:\.[a-zA-Z0-9\-]{1,63}){1,}\b`) + privacyHex = regexp.MustCompile(`\b[A-Fa-f0-9]{24,}\b`) + privacyB64 = regexp.MustCompile(`\b[A-Za-z0-9+/]{28,}={0,2}\b`) + privacyKey = regexp.MustCompile(`(?i)(password|passwd|token|secret|apikey|api[_-]?key|bearer|authorization)\s*[:=]\s*\S+`) + privacyUsrAt = regexp.MustCompile(`[A-Za-z0-9._-]+@[A-Za-z0-9.-]+`) +) + +// privacyGlyphs is a small set of dense unicode shade characters. By +// cycling through them the redacted range looks like diffused noise +// rather than a flat mask. +var privacyGlyphs = []rune{'▚', '▞', '▓', '▒'} + +// privacyMask returns a redaction string the same visual length as src. +func privacyMask(src string) string { + rs := []rune(src) + out := make([]rune, len(rs)) + for i, r := range rs { + if r == ' ' || r == '\t' || r == '\n' { + out[i] = r + continue + } + out[i] = privacyGlyphs[i%len(privacyGlyphs)] + } + return string(out) +} + +// privacyRedact scrubs every sensitive pattern from the given line. The +// function is intentionally line-scoped — callers apply it row by row so +// that multi-line ANSI layouts survive the substitution. +func privacyRedact(line string) string { + if line == "" { + return line + } + replace := func(re *regexp.Regexp, s string) string { + return re.ReplaceAllStringFunc(s, privacyMask) + } + // Order matters: scrub the longest/most specific patterns first so + // later passes don't hit already-masked text. + line = privacyKey.ReplaceAllStringFunc(line, func(m string) string { + // Keep the label (password/token/etc) but mask the value. + idx := strings.IndexAny(m, "=:") + if idx < 0 { + return privacyMask(m) + } + return m[:idx+1] + privacyMask(strings.TrimLeft(m[idx+1:], " ")) + }) + line = replace(privacyB64, line) + line = replace(privacyHex, line) + line = replace(privacyMAC, line) + line = replace(privacyIPv4, line) + line = replace(privacyIPv6, line) + line = replace(privacyUsrAt, line) + line = replace(privacyHost, line) + return line +} + +// applyPrivacyMultiline redacts every line independently. Safe to call +// on ANSI-styled output — masks only the literal runs a regex matches. +func applyPrivacyMultiline(s string) string { + if s == "" { + return s + } + lines := strings.Split(s, "\n") + for i, ln := range lines { + lines[i] = privacyRedact(ln) + } + return strings.Join(lines, "\n") +} diff --git a/internal/cli/monitor_ssh.go b/internal/cli/monitor_ssh.go new file mode 100644 index 0000000..31f2010 --- /dev/null +++ b/internal/cli/monitor_ssh.go @@ -0,0 +1,466 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/hinshun/vt10x" + + "pxmon/internal/cluster" +) + +// sshStartedMsg is dispatched when an embedded SSH session finishes dialing. +type sshStartedMsg struct { + session *cluster.InteractiveSession + cluster cluster.Cluster + cols int + rows int + err error +} + +// sshChunkMsg delivers a chunk of remote PTY output to the model. +type sshChunkMsg struct { + data []byte + err error +} + +// sshClosedMsg signals that the embedded session was torn down. +type sshClosedMsg struct { + err error +} + +// sshTickMsg throttles vt10x → view repaints while output is flowing. +type sshTickMsg struct{} + +const ( + sshMinCols = 20 + sshMinRows = 5 + sshChunkBuffer = 16384 + sshRepaintInterval = 33 * time.Millisecond +) + +// sshEmbeddedDims returns the usable grid size for the SSH panel. +func (m monitorModel) sshEmbeddedDims() (int, int) { + w := m.width + if w <= 0 { + w = 120 + } + h := m.height + if h <= 0 { + h = 32 + } + cols := w - 4 + rows := h - 4 + if cols < sshMinCols { + cols = sshMinCols + } + if rows < sshMinRows { + rows = sshMinRows + } + return cols, rows +} + +// startEmbeddedSSHCmd kicks off an SSH dial in a goroutine and returns +// the started session through an sshStartedMsg. +func (m monitorModel) startEmbeddedSSHCmd(selector string) tea.Cmd { + svc := m.svc + cols, rows := m.sshEmbeddedDims() + c, err := svc.Get(selector) + if err != nil { + return func() tea.Msg { return sshStartedMsg{err: err} } + } + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + sess, err := svc.StartInteractiveShell(ctx, c.ID, cols, rows, "xterm-256color") + if err != nil { + return sshStartedMsg{err: err, cluster: c} + } + return sshStartedMsg{ + session: sess, + cluster: c, + cols: cols, + rows: rows, + } + } +} + +// readSSHChunkCmd reads the next chunk from the SSH session in a goroutine. +func readSSHChunkCmd(sess *cluster.InteractiveSession) tea.Cmd { + if sess == nil { + return nil + } + return func() tea.Msg { + buf := make([]byte, sshChunkBuffer) + n, err := sess.Read(buf) + if n > 0 { + data := make([]byte, n) + copy(data, buf[:n]) + return sshChunkMsg{data: data, err: err} + } + return sshChunkMsg{err: err} + } +} + +// sshRepaintTickCmd schedules a throttled repaint while output flows. +func sshRepaintTickCmd() tea.Cmd { + return tea.Tick(sshRepaintInterval, func(_ time.Time) tea.Msg { + return sshTickMsg{} + }) +} + +// enterEmbeddedSSH switches the model into embedded SSH view for the +// freshly dialed session. +func (m *monitorModel) enterEmbeddedSSH(msg sshStartedMsg) { + cols := msg.cols + rows := msg.rows + if cols < sshMinCols { + cols = sshMinCols + } + if rows < sshMinRows { + rows = sshMinRows + } + vt := vt10x.New(vt10x.WithSize(cols, rows)) + m.sshMode = true + m.sshSess = msg.session + m.sshCluster = msg.cluster + m.sshVT = vt + m.sshCols = cols + m.sshRows = rows + m.sshClosed = false + m.sshErr = "" + m.sshPendingRepaint = false +} + +// exitEmbeddedSSH tears down the embedded session and clears state. +func (m *monitorModel) exitEmbeddedSSH(reason string) { + if m.sshSess != nil { + _ = m.sshSess.Close() + } + m.sshSess = nil + m.sshVT = nil + m.sshMode = false + m.sshClosed = true + m.sshPendingRepaint = false + // Return to pxmon console so the user lands where they launched from. + m.termMode = true + m.termFull = false + if reason != "" { + m.setStatus(reason) + } +} + +// writeSSHInput pushes a byte slice to the SSH session's stdin. +func (m monitorModel) writeSSHInput(p []byte) { + if m.sshSess == nil || len(p) == 0 { + return + } + _, _ = m.sshSess.Write(p) +} + +// handleSSHKey routes TUI key events into the remote PTY stdin. +func (m monitorModel) handleSSHKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + // Escape hatch: Ctrl+] closes the embedded session. + if v.Type == tea.KeyCtrlCloseBracket { + m.exitEmbeddedSSH(fmt.Sprintf("ssh closed (%s)", m.sshCluster.Name)) + m.appendConsoleOutput(fmt.Sprintf("ssh session closed (%s)", m.sshCluster.Name)) + return m, nil + } + // Scrollback controls — mirror the pxmon console (alt+↑/↓, pgup/pgdn, + // alt+home/end). Alt+Shift+↑/↓ jump 10 lines at a time for fast review. + // These never reach the remote PTY. + switch v.String() { + case "alt+up": + m.scrollSSH(1) + return m, nil + case "alt+down": + m.scrollSSH(-1) + return m, nil + case "alt+shift+up": + m.scrollSSH(10) + return m, nil + case "alt+shift+down": + m.scrollSSH(-10) + return m, nil + case "pgup": + m.scrollSSH(m.sshRows - 1) + return m, nil + case "pgdown": + m.scrollSSH(-(m.sshRows - 1)) + return m, nil + case "alt+home": + m.sshScrollOffset = len(m.sshScrollback) + m.clampSSHScroll() + return m, nil + case "alt+end": + m.sshScrollOffset = 0 + return m, nil + case "alt+p": + m.privacyMode = !m.privacyMode + if m.privacyMode { + m.setStatus("privacy: on") + } else { + m.setStatus("privacy: off") + } + return m, nil + } + // Any other input resumes live view if the user was scrolling back. + if m.sshScrollOffset != 0 { + m.sshScrollOffset = 0 + } + payload := keyMsgToPTY(v) + if len(payload) == 0 { + return m, nil + } + m.writeSSHInput(payload) + return m, nil +} + +// captureSSHScrollback appends remote output (with ANSI sequences stripped) +// to the scrollback buffer so the user can scroll through past output even +// though vt10x does not retain a scroll history of its own. +func (m *monitorModel) captureSSHScrollback(data []byte) { + if len(data) == 0 { + return + } + text := stripANSI(string(data)) + // Treat stand-alone CR as a rewrite of the current line; drop content + // before the CR to avoid duplicating progress-bar style updates. + combined := m.sshScrollPending + text + combined = strings.ReplaceAll(combined, "\r\n", "\n") + lines := strings.Split(combined, "\n") + // Last element is either a trailing newline ("") or a partial line; stash. + m.sshScrollPending = lines[len(lines)-1] + lines = lines[:len(lines)-1] + for _, ln := range lines { + if idx := strings.LastIndex(ln, "\r"); idx >= 0 { + ln = ln[idx+1:] + } + m.sshScrollback = append(m.sshScrollback, ln) + } + const maxScrollback = 5000 + if len(m.sshScrollback) > maxScrollback { + m.sshScrollback = m.sshScrollback[len(m.sshScrollback)-maxScrollback:] + } + // A new chunk means more history arrived behind the current scroll + // window — adjust offset so the user keeps looking at the same line. + if m.sshScrollOffset > 0 { + m.sshScrollOffset += len(lines) + m.clampSSHScroll() + } +} + +func (m *monitorModel) scrollSSH(delta int) { + m.sshScrollOffset += delta + m.clampSSHScroll() +} + +func (m *monitorModel) clampSSHScroll() { + maxOffset := len(m.sshScrollback) - m.sshRows + if maxOffset < 0 { + maxOffset = 0 + } + if m.sshScrollOffset > maxOffset { + m.sshScrollOffset = maxOffset + } + if m.sshScrollOffset < 0 { + m.sshScrollOffset = 0 + } +} + +// handleSSHResize adjusts the vt10x grid and notifies the remote side. +func (m *monitorModel) handleSSHResize() { + if !m.sshMode || m.sshVT == nil || m.sshSess == nil { + return + } + cols, rows := m.sshEmbeddedDims() + if cols == m.sshCols && rows == m.sshRows { + return + } + m.sshCols = cols + m.sshRows = rows + m.sshVT.Resize(cols, rows) + _ = m.sshSess.Resize(cols, rows) +} + +// renderSSHView produces the TUI view for the embedded SSH session. +func (m monitorModel) renderSSHView() string { + header := accentStyle.Bold(true).Render(fmt.Sprintf(" ssh://%s@%s:%d (%s) ", m.sshCluster.User, m.sshCluster.Host, m.sshCluster.Port, m.sshCluster.Name)) + hint := dimStyle.Render("Ctrl+] detach · alt+↑/↓ scroll · pgup/pgdn page · alt+end live") + if m.sshScrollOffset > 0 { + hint = warnStyle.Render(fmt.Sprintf("↑ scrolled +%d (alt+end to follow)", m.sshScrollOffset)) + + dimStyle.Render(" Ctrl+] detach") + } + + body := m.renderVTBody() + + panel := lipgloss.NewStyle(). + BorderStyle(thinBorder). + BorderForeground(ccAccent). + Padding(0, 1). + Render(body) + + status := dimStyle.Render(strings.TrimSpace("status: " + m.statusMsg)) + return lipgloss.JoinVertical(lipgloss.Left, header+" "+hint, panel, status) +} + +// renderVTBody iterates the vt10x grid and emits a styled string block. +// When the user has scrolled back it instead renders a window of the +// ANSI-stripped scrollback buffer. +func (m monitorModel) renderVTBody() string { + if m.sshVT == nil { + return dimStyle.Render("(session not ready)") + } + if m.sshScrollOffset > 0 { + return m.renderScrollbackBody() + } + vt := m.sshVT + vt.Lock() + cols, rows := vt.Size() + cur := vt.Cursor() + cursorVisible := vt.CursorVisible() + + var lines []string + for y := 0; y < rows; y++ { + line := renderVTRow(vt, y, cols, cursorVisible, cur.X, cur.Y) + lines = append(lines, line) + } + vt.Unlock() + return strings.Join(lines, "\n") +} + +// renderScrollbackBody paints a page of the captured scrollback buffer when +// the user is browsing history. Every row is padded to the full grid width +// so the panel keeps its live dimensions — otherwise lipgloss sizes the +// border to the longest line and the view visibly collapses. +func (m monitorModel) renderScrollbackBody() string { + rows := m.sshRows + cols := m.sshCols + if rows <= 0 { + rows = sshMinRows + } + if cols <= 0 { + cols = sshMinCols + } + end := len(m.sshScrollback) - m.sshScrollOffset + if end < 0 { + end = 0 + } + start := end - rows + if start < 0 { + start = 0 + } + blank := strings.Repeat(" ", cols) + out := make([]string, 0, rows) + pad := func(line string) string { + rs := []rune(line) + if len(rs) > cols { + rs = rs[:cols] + } + if len(rs) < cols { + return string(rs) + strings.Repeat(" ", cols-len(rs)) + } + return string(rs) + } + for i := start; i < end; i++ { + out = append(out, pad(m.sshScrollback[i])) + } + for len(out) < rows { + out = append(out, blank) + } + return strings.Join(out, "\n") +} + +// renderVTRow renders a single grid row as a styled string. +func renderVTRow(vt vt10x.Terminal, row, cols int, cursorVisible bool, cursorX, cursorY int) string { + var b strings.Builder + var ( + runRunes strings.Builder + runFG vt10x.Color = vt10x.DefaultFG + runBG vt10x.Color = vt10x.DefaultBG + runStart = true + ) + flush := func() { + if runRunes.Len() == 0 { + return + } + style := styleForColors(runFG, runBG) + b.WriteString(style.Render(runRunes.String())) + runRunes.Reset() + } + for x := 0; x < cols; x++ { + cell := vt.Cell(x, row) + ch := cell.Char + if ch == 0 { + ch = ' ' + } + fg := cell.FG + bg := cell.BG + isCursor := cursorVisible && x == cursorX && row == cursorY + if runStart { + runFG = fg + runBG = bg + runStart = false + } + if isCursor { + // Emit the current run first, then paint the cursor cell with + // an explicit, always-visible color so the caret shows even on + // empty cells with default FG/BG (where a plain invert would + // collapse to the same color). + flush() + cursorStyle := lipgloss.NewStyle(). + Background(ccAccent). + Foreground(lipgloss.Color("#101010")). + Bold(true) + b.WriteString(cursorStyle.Render(string(ch))) + runFG = fg + runBG = bg + continue + } + if fg != runFG || bg != runBG { + flush() + runFG = fg + runBG = bg + } + runRunes.WriteRune(ch) + } + flush() + return b.String() +} + +// styleForColors returns a lipgloss style for the given vt10x colors. +func styleForColors(fg, bg vt10x.Color) lipgloss.Style { + style := lipgloss.NewStyle() + if c, ok := vtColorToLipgloss(fg); ok { + style = style.Foreground(c) + } + if c, ok := vtColorToLipgloss(bg); ok { + style = style.Background(c) + } + return style +} + +// vtColorToLipgloss maps a vt10x color to a lipgloss color. Returns ok=false +// for default so the caller leaves the attribute unset. +func vtColorToLipgloss(c vt10x.Color) (lipgloss.TerminalColor, bool) { + if c == vt10x.DefaultFG || c == vt10x.DefaultBG || c == vt10x.DefaultCursor { + return nil, false + } + // ANSI basic 16 + if c < 16 { + return lipgloss.Color(fmt.Sprintf("%d", uint32(c))), true + } + // 256-color palette + if c < 256 { + return lipgloss.Color(fmt.Sprintf("%d", uint32(c))), true + } + // Truecolor (24-bit): stored in low 24 bits. + r := (uint32(c) >> 16) & 0xff + g := (uint32(c) >> 8) & 0xff + bl := uint32(c) & 0xff + return lipgloss.Color(fmt.Sprintf("#%02x%02x%02x", r, g, bl)), true +} diff --git a/internal/cli/monitor_usage.go b/internal/cli/monitor_usage.go new file mode 100644 index 0000000..49aa69f --- /dev/null +++ b/internal/cli/monitor_usage.go @@ -0,0 +1,241 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +type monitorUsageMsg struct { + snap cluster.UsageSnapshot + err error +} + +func (m monitorModel) fetchUsageCmd(rng history.RangeShortcut) tea.Cmd { + svc := m.svc + selector := m.cluster.ID + duPath := "/" + return func() tea.Msg { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, duPath) + return monitorUsageMsg{snap: snap, err: err} + } +} + +func (m monitorModel) handleUsageKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { + switch v.String() { + case "o", "esc": + m.view = viewOverview + return m, nil + case "n": + m.view = viewNetwork + return m, nil + case "c": + m.view = viewClusters + return m, nil + case "s": + m.view = viewSettings + return m, nil + case "r", "R": + m.usageLoading = true + m.setStatus("usage: refreshing") + return m, m.fetchUsageCmd(m.usageRange) + case "1": + m.usageRange = history.RangeLive + m.usageLoading = true + m.setStatus("usage: live") + return m, m.fetchUsageCmd(m.usageRange) + case "2": + m.usageRange = history.RangeHour + m.usageLoading = true + m.setStatus("usage: last 1h") + return m, m.fetchUsageCmd(m.usageRange) + case "3": + m.usageRange = history.RangeDay + m.usageLoading = true + m.setStatus("usage: last 24h") + return m, m.fetchUsageCmd(m.usageRange) + case "4": + m.usageRange = history.RangeMonth + m.usageLoading = true + m.setStatus("usage: last 30d") + return m, m.fetchUsageCmd(m.usageRange) + case "5": + m.usageRange = history.RangeAll + m.usageLoading = true + m.setStatus("usage: all time") + return m, m.fetchUsageCmd(m.usageRange) + } + return m, nil +} + +func (m monitorModel) renderUsageDashboard(width int) string { + title := titleStyle.Render("━━ Usage / billing ━━") + help := dimStyle.Render("range: 1=live 2=1h 3=1d 4=1mo 5=all r refresh o back") + + if m.usageLoading && m.usageSnap == nil { + body := dimStyle.Render("loading usage snapshot…") + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) + } + + if m.usageSnap == nil { + if m.usageErr != nil { + body := critStyle.Render("error: " + m.usageErr.Error()) + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) + } + body := dimStyle.Render("press u to load usage") + return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) + } + + snap := *m.usageSnap + rng := m.usageRange + + lines := []string{title, help, ""} + if m.usageLoading { + lines = append(lines, dimStyle.Render("(refreshing…)")) + } + if m.usageErr != nil { + lines = append(lines, critStyle.Render("error: "+m.usageErr.Error())) + } + + lines = append(lines, + fmt.Sprintf("%s %s %s %s %s %s %s %d", + dimStyle.Render("cluster:"), accentStyle.Render(snap.ClusterName), + dimStyle.Render("range:"), brightStyle.Render(rng.Label()), + dimStyle.Render("cpu:"), brightStyle.Render(fmt.Sprintf("%.1f%%", snap.Live.CPU.UsagePercent)), + dimStyle.Render("procs:"), snap.Top.TotalProcs, + ), + fmt.Sprintf("%s %s %s %s", + dimStyle.Render("ram:"), brightStyle.Render(fmt.Sprintf("%.1f%%", snap.Live.Memory.UsedPercent)), + dimStyle.Render("host:"), accentStyle.Render(snap.Live.Host.Hostname), + ), + "", + titleStyle.Render("── Network (uplink, max(Rx,Tx)) ──"), + ) + + if snap.HistoryError != "" { + lines = append(lines, warnStyle.Render("history: "+snap.HistoryError)) + } + lines = append(lines, + fmt.Sprintf(" %s %s %s %s %s %s (%d samples)", + dimStyle.Render("P95:"), okStyle.Render(formatMbpsHuman(snap.P95TotalMbps)), + dimStyle.Render("max:"), brightStyle.Render(formatMbpsHuman(snap.MaxTotalMbps)), + dimStyle.Render("avg:"), brightStyle.Render(formatMbpsHuman(snap.AvgTotalMbps)), + len(snap.NodeSeries), + ), + ) + if snap.TopIfaceName != "" { + lines = append(lines, + fmt.Sprintf(" %s %s (avg %s)", + dimStyle.Render("uplink:"), + accentStyle.Render(snap.TopIfaceName), + formatMbpsHuman(snap.TopIfaceMbps)), + ) + } + + // Sparkline from node series + if len(snap.NodeSeries) > 1 { + lines = append(lines, " "+dimStyle.Render("series:")+" "+renderNodeSparkline(snap.NodeSeries, 40)) + } + + lines = append(lines, "", titleStyle.Render("── Top processes (by CPU) ──")) + if snap.TopError != "" { + lines = append(lines, warnStyle.Render("top: "+snap.TopError)) + } else { + for i, p := range snap.Top.TopByCPU { + if i >= 6 { + break + } + lines = append(lines, fmt.Sprintf(" %5d %-10s %s%% %8s %s", + p.PID, + clipRight(p.User, 10), + brightStyle.Render(fmt.Sprintf("%5.1f", p.CPUPercent)), + humanBytesUint(p.RSSBytes), + accentStyle.Render(clipRight(p.Command, 36)))) + } + } + + lines = append(lines, "", titleStyle.Render("── Top processes (by RAM) ──")) + if snap.TopError == "" { + for i, p := range snap.Top.TopByMemory { + if i >= 6 { + break + } + lines = append(lines, fmt.Sprintf(" %5d %-10s %8s %s", + p.PID, + clipRight(p.User, 10), + brightStyle.Render(humanBytesUint(p.RSSBytes)), + accentStyle.Render(clipRight(p.Command, 44)))) + } + } + + lines = append(lines, "", titleStyle.Render("── Top folders ──")) + if snap.DUError != "" { + lines = append(lines, warnStyle.Render("du: "+snap.DUError)) + } else { + for i, d := range snap.DU.Entries { + if i >= 8 { + break + } + lines = append(lines, fmt.Sprintf(" %10s %s", + brightStyle.Render(humanBytesUint(d.Bytes)), + accentStyle.Render(d.Path))) + } + if snap.DU.Truncated { + lines = append(lines, warnStyle.Render(" (scan truncated by timeout)")) + } + } + + return strings.Join(lines, "\n") +} + +func renderNodeSparkline(series []history.NodeSamplePoint, width int) string { + if len(series) == 0 || width <= 0 { + return "" + } + maxV := 0.0 + for _, p := range series { + if p.TotalMbps > maxV { + maxV = p.TotalMbps + } + } + if maxV <= 0 { + return strings.Repeat("·", width) + } + step := len(series) / width + if step < 1 { + step = 1 + } + runes := []rune{' ', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'} + var b strings.Builder + for i := 0; i < len(series); i += step { + v := series[i].TotalMbps + idx := int((v / maxV) * float64(len(runes)-1)) + if idx < 0 { + idx = 0 + } + if idx >= len(runes) { + idx = len(runes) - 1 + } + b.WriteRune(runes[idx]) + } + return b.String() +} + +func clipRight(s string, n int) string { + if len(s) <= n { + return s + } + if n <= 3 { + return s[:n] + } + return s[:n-3] + "..." +} diff --git a/internal/cli/shell.go b/internal/cli/shell.go new file mode 100644 index 0000000..13c31f9 --- /dev/null +++ b/internal/cli/shell.go @@ -0,0 +1,286 @@ +package cli + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "strings" + "unicode" + + "pxmon/internal/cluster" +) + +type shellStatus struct { + ActiveCluster string + ClusterCount int + LoadError error +} + +func (a *App) runShell(configPath string, jsonOut bool) int { + status := a.loadShellStatus(configPath) + a.printShellBanner(status, configPath, jsonOut) + + history := make([]string, 0, 64) + reader := bufio.NewReader(os.Stdin) + + for { + status = a.loadShellStatus(configPath) + fmt.Fprint(a.out, status.prompt()) + + line, err := reader.ReadString('\n') + if err != nil { + if errors.Is(err, io.EOF) { + fmt.Fprintln(a.out) + return 0 + } + fmt.Fprintf(a.err, "shell input error: %v\n", err) + return 1 + } + + line = strings.TrimSpace(line) + if line == "" { + continue + } + history = append(history, line) + + args, parseErr := parseShellArgs(line) + if parseErr != nil { + fmt.Fprintf(a.err, "parse error: %v\n", parseErr) + continue + } + if len(args) == 0 { + continue + } + if strings.EqualFold(args[0], "pxmon") || strings.EqualFold(args[0], "pxmon") { + args = args[1:] + } + if len(args) == 0 { + continue + } + + switch shellToken(args[0]) { + case "exit", "quit", "q", ":q": + return 0 + case "clear", "cls": + fmt.Fprint(a.out, "\x1b[2J\x1b[H") + continue + case "history": + for i, h := range history { + fmt.Fprintf(a.out, "%3d %s\n", i+1, h) + } + continue + case "help", "?": + a.printShellHelp() + continue + case "status", "dashboard": + a.printShellStatus(status, configPath, jsonOut) + continue + } + + cmd := expandShellCommand(args) + code := a.runRootCommand(cmd, configPath, jsonOut, false) + if code != 0 { + fmt.Fprintf(a.err, "[exit %d]\n", code) + } + } +} + +func (a *App) loadShellStatus(configPath string) shellStatus { + status := shellStatus{} + + store, err := cluster.NewStore(configPath) + if err != nil { + status.LoadError = err + return status + } + svc := cluster.NewService(store) + + clusters, _, err := svc.List() + if err != nil { + status.LoadError = err + return status + } + status.ClusterCount = len(clusters) + + current, err := svc.Current() + if err == nil { + status.ActiveCluster = current.Name + return status + } + if !errors.Is(err, cluster.ErrNoActiveCluster) { + status.LoadError = err + } + + return status +} + +func (s shellStatus) prompt() string { + active := strings.TrimSpace(s.ActiveCluster) + if active == "" { + active = "none" + } + active = strings.ReplaceAll(active, "]", "_") + return fmt.Sprintf("pxmon[%s]> ", active) +} + +func (a *App) printShellBanner(status shellStatus, configPath string, jsonOut bool) { + fmt.Fprintln(a.out, "+----------------------------------------------------------------+") + fmt.Fprintln(a.out, "| pxmon interactive shell |") + fmt.Fprintln(a.out, "| slash: /help /clusters /use /stats [name] /network |") + fmt.Fprintln(a.out, "| exit: /quit or /q |") + fmt.Fprintln(a.out, "+----------------------------------------------------------------+") + a.printShellStatus(status, configPath, jsonOut) + fmt.Fprintln(a.out) +} + +func (a *App) printShellStatus(status shellStatus, configPath string, jsonOut bool) { + active := strings.TrimSpace(status.ActiveCluster) + if active == "" { + active = "none" + } + + jsonState := "off" + if jsonOut { + jsonState = "on" + } + + fmt.Fprintf(a.out, "Clusters: %d | Active: %s | JSON: %s\n", status.ClusterCount, active, jsonState) + if strings.TrimSpace(configPath) != "" { + fmt.Fprintf(a.out, "Config: %s\n", configPath) + } + if status.LoadError != nil { + fmt.Fprintf(a.out, "Status error: %v\n", status.LoadError) + } +} + +func (a *App) printShellHelp() { + fmt.Fprintln(a.out, "Shell commands:") + fmt.Fprintln(a.out, " help Show this help") + fmt.Fprintln(a.out, " status Show shell status") + fmt.Fprintln(a.out, " history Show command history") + fmt.Fprintln(a.out, " clear Clear screen") + fmt.Fprintln(a.out, " exit | quit | q Exit shell") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Shortcuts:") + fmt.Fprintln(a.out, " /clusters Alias for 'cluster list'") + fmt.Fprintln(a.out, " /stats [name] Alias for 'cluster stats [name]'") + fmt.Fprintln(a.out, " /monitor [name] Alias for 'cluster stats [name]'") + fmt.Fprintln(a.out, " /network [name] Open network-focused TUI") + fmt.Fprintln(a.out, " /connect ... Alias for 'cluster connect ...'") + fmt.Fprintln(a.out, " /use Alias for 'cluster use '") + fmt.Fprintln(a.out, " /alert ... Alias for 'cluster alert ...'") + fmt.Fprintln(a.out) + fmt.Fprintln(a.out, "Examples:") + fmt.Fprintln(a.out, " /clusters") + fmt.Fprintln(a.out, " /connect --name eu-1 --host 10.0.0.10 --user root --auth key --key-path ~/.ssh/id_ed25519") + fmt.Fprintln(a.out, " /bootstrap eu-1") + fmt.Fprintln(a.out, " /stats eu-1") + fmt.Fprintln(a.out, " /network eu-1") + fmt.Fprintln(a.out, " /alert set eu-1 --net-mbps 300 --ram 90 --disk 90") + fmt.Fprintln(a.out, " /alert set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup") +} + +func shellToken(v string) string { + token := strings.TrimSpace(strings.ToLower(v)) + if strings.HasPrefix(token, "/") { + token = strings.TrimPrefix(token, "/") + } + return token +} + +func expandShellCommand(args []string) []string { + if len(args) == 0 { + return args + } + + out := append([]string(nil), args...) + cmd := shellToken(out[0]) + if cmd == "" { + return out + } + out[0] = cmd + + switch cmd { + case "clusters", "nodes": + return append([]string{"cluster", "list"}, out[1:]...) + case "stats", "monitor", "watch": + return append([]string{"cluster", "stats"}, out[1:]...) + case "alerts": + return append([]string{"cluster", "alert"}, out[1:]...) + } + + if isClusterShortcut(cmd) { + return append([]string{"cluster", cmd}, out[1:]...) + } + + return out +} + +func isClusterShortcut(cmd string) bool { + switch cmd { + case "connect", "add", "list", "ls", "show", "get", "current", "use", + "ping", "check", "bootstrap", "disconnect", "remove", "rm", "alert": + return true + default: + return false + } +} + +func parseShellArgs(line string) ([]string, error) { + line = strings.TrimSpace(line) + if line == "" { + return nil, nil + } + + args := make([]string, 0, 8) + var current strings.Builder + var quote rune + escaped := false + tokenStarted := false + + flush := func() { + if tokenStarted { + args = append(args, current.String()) + current.Reset() + tokenStarted = false + } + } + + for _, r := range line { + switch { + case escaped: + current.WriteRune(r) + escaped = false + tokenStarted = true + case r == '\\': + escaped = true + tokenStarted = true + case quote != 0: + if r == quote { + quote = 0 + } else { + current.WriteRune(r) + } + tokenStarted = true + case r == '\'' || r == '"': + quote = r + tokenStarted = true + case unicode.IsSpace(r): + flush() + default: + current.WriteRune(r) + tokenStarted = true + } + } + + if escaped { + return nil, errors.New("unterminated escape at end of command") + } + if quote != 0 { + return nil, errors.New("unterminated quoted string") + } + flush() + return args, nil +} diff --git a/internal/cli/usage.go b/internal/cli/usage.go new file mode 100644 index 0000000..95f0f38 --- /dev/null +++ b/internal/cli/usage.go @@ -0,0 +1,414 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "text/tabwriter" + "time" + + "pxmon/internal/agent" + "pxmon/internal/cluster" + "pxmon/internal/history" +) + +func (a *App) runClusterUsage(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster usage", flag.ContinueOnError) + fs.SetOutput(a.err) + rangeStr := fs.String("range", "live", "Time window: live|1h|1d|1mo|all") + duPath := fs.String("du", "/", "Root directory for top-folder scan") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + selector := "" + if fs.NArg() > 0 { + selector = fs.Arg(0) + } + + rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr)) + if !ok { + fmt.Fprintf(a.err, "usage: invalid --range %q (expected live|1h|1d|1mo|all)\n", *rangeStr) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + + snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, *duPath) + if err != nil { + fmt.Fprintf(a.err, "cluster usage: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, snap) + return 0 + } + + printUsageSnapshot(a.out, snap, rng) + return 0 +} + +func (a *App) runClusterTraffic(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster traffic", flag.ContinueOnError) + fs.SetOutput(a.err) + rangeStr := fs.String("range", "1h", "Time window: 1h|1d|1mo|all") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + selector := "" + if fs.NArg() > 0 { + selector = fs.Arg(0) + } + + rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr)) + if !ok { + fmt.Fprintf(a.err, "traffic: invalid --range %q\n", *rangeStr) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, "") + if err != nil { + fmt.Fprintf(a.err, "cluster traffic: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "cluster": snap.ClusterName, + "range": rng.Label(), + "p95_mbps": snap.P95TotalMbps, + "max_mbps": snap.MaxTotalMbps, + "avg_mbps": snap.AvgTotalMbps, + "samples": len(snap.NodeSeries), + "top_iface": snap.TopIfaceName, + "top_iface_mbps": snap.TopIfaceMbps, + "history_error": snap.HistoryError, + }) + return 0 + } + + fmt.Fprintf(a.out, "%s %s %s\n", colorLabel("Cluster:"), colorAccent(snap.ClusterName), colorMuted("("+rng.Label()+")")) + if snap.HistoryError != "" { + fmt.Fprintf(a.out, "%s %s\n", colorWarn("History:"), snap.HistoryError) + } + fmt.Fprintf(a.out, "%s %s %s %d samples\n", + colorLabel("P95: "), + colorOK(formatMbpsHuman(snap.P95TotalMbps)), + colorMuted("over"), + len(snap.NodeSeries), + ) + fmt.Fprintf(a.out, "%s %s %s %s\n", + colorLabel("Max: "), + colorValue(formatMbpsHuman(snap.MaxTotalMbps)), + colorMuted("avg:"), + colorInfo(formatMbpsHuman(snap.AvgTotalMbps)), + ) + if snap.TopIfaceName != "" { + fmt.Fprintf(a.out, "%s %s %s %s\n", + colorLabel("Uplink:"), + colorAccent(snap.TopIfaceName), + colorMuted("avg max(Rx,Tx):"), + colorValue(formatMbpsHuman(snap.TopIfaceMbps)), + ) + } + return 0 +} + +func (a *App) runClusterGraph(svc *cluster.Service, args []string, jsonOut bool) int { + fs := flag.NewFlagSet("cluster graph", flag.ContinueOnError) + fs.SetOutput(a.err) + rangeStr := fs.String("range", "1d", "Time window: 1h|1d|1mo|all") + outPath := fs.String("out", "", "Output PNG path (default: ./-.png)") + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + selector := "" + if fs.NArg() > 0 { + selector = fs.Arg(0) + } + + rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr)) + if !ok { + fmt.Fprintf(a.err, "graph: invalid --range %q\n", *rangeStr) + return 2 + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, "") + if err != nil { + fmt.Fprintf(a.err, "cluster graph: %v\n", err) + return 1 + } + + png, err := cluster.RenderUsageChartPNG(snap, "") + if err != nil { + fmt.Fprintf(a.err, "cluster graph: render: %v\n", err) + return 1 + } + + dest := strings.TrimSpace(*outPath) + if dest == "" { + dest = filepath.Join(".", fmt.Sprintf("%s-%s.png", sanitizeFilename(snap.ClusterName), rng)) + } + if err := os.WriteFile(dest, png, 0o600); err != nil { + fmt.Fprintf(a.err, "cluster graph: write: %v\n", err) + return 1 + } + + if jsonOut { + _ = writeJSON(a.out, map[string]any{ + "path": dest, + "bytes": len(png), + "p95_mbps": snap.P95TotalMbps, + "samples": len(snap.NodeSeries), + }) + return 0 + } + + fmt.Fprintf(a.out, "%s %s\n", colorLabel("Saved:"), colorAccent(dest)) + fmt.Fprintf(a.out, "%s %s %s %d samples\n", + colorLabel("P95: "), + colorOK(formatMbpsHuman(snap.P95TotalMbps)), + colorMuted("over"), + len(snap.NodeSeries), + ) + return 0 +} + +func printUsageSnapshot(w io.Writer, snap cluster.UsageSnapshot, rng history.RangeShortcut) { + fmt.Fprintf(w, "%s %s %s\n", colorLabel("Cluster:"), colorAccent(snap.ClusterName), colorMuted("("+rng.Label()+")")) + fmt.Fprintf(w, "%s %s %s\n", colorLabel("Host: "), colorAccent(snap.Live.Host.Hostname), colorMuted(snap.Live.Host.OS+"/"+snap.Live.Host.Arch)) + + fmt.Fprintf(w, "%s %s %s %s %s %s\n", + colorLabel("CPU:"), + colorValue(fmt.Sprintf("%.1f%%", snap.Live.CPU.UsagePercent)), + colorLabel("RAM:"), + colorValue(fmt.Sprintf("%.1f%%", snap.Live.Memory.UsedPercent)), + colorLabel("Procs:"), + colorInfo(fmt.Sprintf("%d", snap.Top.TotalProcs)), + ) + + fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Network ━━")) + if snap.HistoryError != "" { + fmt.Fprintf(w, "%s %s\n", colorWarn("history:"), snap.HistoryError) + } + fmt.Fprintf(w, " %s %s %s %s %s %s (%d samples)\n", + colorLabel("P95:"), colorOK(formatMbpsHuman(snap.P95TotalMbps)), + colorLabel("max:"), colorValue(formatMbpsHuman(snap.MaxTotalMbps)), + colorLabel("avg:"), colorInfo(formatMbpsHuman(snap.AvgTotalMbps)), + len(snap.NodeSeries), + ) + if snap.TopIfaceName != "" { + fmt.Fprintf(w, " %s %s %s %s\n", + colorLabel("uplink:"), + colorAccent(snap.TopIfaceName), + colorMuted("avg max(Rx,Tx)"), + colorValue(formatMbpsHuman(snap.TopIfaceMbps)), + ) + } + + if snap.TopError != "" { + fmt.Fprintf(w, "\n%s %s\n", colorWarn("top:"), snap.TopError) + } else { + fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Top processes (by CPU) ━━")) + printProcessTable(w, snap.Top.TopByCPU) + fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Top processes (by RAM) ━━")) + printProcessTable(w, snap.Top.TopByMemory) + } + + if snap.DUError != "" { + fmt.Fprintf(w, "\n%s %s\n", colorWarn("du:"), snap.DUError) + } else { + fmt.Fprintf(w, "\n%s %s\n", colorHeader("━━ Top folders ━━"), colorMuted(snap.DU.Root)) + printFolderTable(w, snap.DU.Entries) + if snap.DU.Truncated { + fmt.Fprintf(w, " %s\n", colorWarn("(scan truncated by timeout)")) + } + } +} + +func printProcessTable(w io.Writer, procs []agent.ProcessStat) { + if len(procs) == 0 { + fmt.Fprintln(w, " (no data)") + return + } + tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0) + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\n", + colorHeader("PID"), colorHeader("USER"), colorHeader("CPU%"), colorHeader("RSS"), colorHeader("CMD")) + for _, p := range procs { + if p.CPUPercent == 0 && p.RSSBytes == 0 { + continue + } + cmd := p.Command + if len(cmd) > 40 { + cmd = cmd[:37] + "..." + } + fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\n", + colorInfo(fmt.Sprintf("%d", p.PID)), + colorMuted(p.User), + colorValue(fmt.Sprintf("%.1f", p.CPUPercent)), + colorValue(humanBytesUint(p.RSSBytes)), + colorAccent(cmd), + ) + } + _ = tw.Flush() +} + +func printFolderTable(w io.Writer, dirs []agent.DirStat) { + if len(dirs) == 0 { + fmt.Fprintln(w, " (no data)") + return + } + tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0) + fmt.Fprintf(tw, " %s\t%s\t%s\n", + colorHeader("SIZE"), colorHeader("FILES"), colorHeader("PATH")) + for _, d := range dirs { + fmt.Fprintf(tw, " %s\t%s\t%s\n", + colorValue(humanBytesUint(d.Bytes)), + colorInfo(fmt.Sprintf("%d", d.Files)), + colorAccent(d.Path), + ) + } + _ = tw.Flush() +} + +func formatMbpsHuman(v float64) string { + switch { + case v >= 1000: + return fmt.Sprintf("%.2f Gbps", v/1000) + case v >= 1: + return fmt.Sprintf("%.1f Mbps", v) + case v > 0: + return fmt.Sprintf("%.0f Kbps", v*1000) + default: + return "0 Mbps" + } +} + +func humanBytesUint(v uint64) string { + const unit = 1024 + if v < unit { + return fmt.Sprintf("%d B", v) + } + div, exp := uint64(unit), 0 + for n := v / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + pre := "KMGTPE" + return fmt.Sprintf("%.2f %ciB", float64(v)/float64(div), pre[exp]) +} + +func sanitizeFilename(name string) string { + name = strings.TrimSpace(name) + if name == "" { + return "cluster" + } + var b bytes.Buffer + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +// formatUsageForBot returns a plain-text rendering (no ANSI) suitable for +// wrapping in a 
 block in Telegram.
+func formatUsageForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Cluster: %s (%s)\n", snap.ClusterName, rng.Label())
+	fmt.Fprintf(&b, "Host:    %s %s/%s\n", snap.Live.Host.Hostname, snap.Live.Host.OS, snap.Live.Host.Arch)
+	fmt.Fprintf(&b, "CPU:  %5.1f%%   RAM: %5.1f%%   Procs: %d\n",
+		snap.Live.CPU.UsagePercent, snap.Live.Memory.UsedPercent, snap.Top.TotalProcs)
+
+	b.WriteString("\n── Network ──\n")
+	if snap.HistoryError != "" {
+		fmt.Fprintf(&b, "history: %s\n", snap.HistoryError)
+	}
+	fmt.Fprintf(&b, "P95 %s · max %s · avg %s  (%d samples)\n",
+		formatMbpsHuman(snap.P95TotalMbps),
+		formatMbpsHuman(snap.MaxTotalMbps),
+		formatMbpsHuman(snap.AvgTotalMbps),
+		len(snap.NodeSeries),
+	)
+	if snap.TopIfaceName != "" {
+		fmt.Fprintf(&b, "uplink: %s (avg max(Rx,Tx): %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps))
+	}
+
+	b.WriteString("\n── Top by CPU ──\n")
+	writeBotProcLine(&b, snap.Top.TopByCPU, 5)
+	b.WriteString("── Top by RAM ──\n")
+	writeBotProcLine(&b, snap.Top.TopByMemory, 5)
+
+	if snap.DUError == "" {
+		fmt.Fprintf(&b, "\n── Top folders (%s) ──\n", snap.DU.Root)
+		for _, d := range snap.DU.Entries {
+			if len(snap.DU.Entries) > 8 {
+				break
+			}
+			fmt.Fprintf(&b, "%10s  %s\n", humanBytesUint(d.Bytes), d.Path)
+		}
+		// if large, take first 8 regardless
+		n := len(snap.DU.Entries)
+		if n > 8 {
+			for i := 0; i < 8; i++ {
+				d := snap.DU.Entries[i]
+				fmt.Fprintf(&b, "%10s  %s\n", humanBytesUint(d.Bytes), d.Path)
+			}
+		}
+	}
+
+	return b.String()
+}
+
+func writeBotProcLine(b *strings.Builder, procs []agent.ProcessStat, n int) {
+	if len(procs) == 0 {
+		b.WriteString("(no data)\n")
+		return
+	}
+	if n > len(procs) {
+		n = len(procs)
+	}
+	for i := 0; i < n; i++ {
+		p := procs[i]
+		cmd := p.Command
+		if len(cmd) > 28 {
+			cmd = cmd[:25] + "..."
+		}
+		user := p.User
+		if len(user) > 8 {
+			user = user[:8]
+		}
+		fmt.Fprintf(b, "%5d %-8s %5.1f%% %8s  %s\n",
+			p.PID, user, p.CPUPercent, humanBytesUint(p.RSSBytes), cmd)
+	}
+}
+
+// writeJSON is used by subcommands for --json output. Declared in app.go.
+var _ = json.Marshal
diff --git a/internal/cli/winch_unix.go b/internal/cli/winch_unix.go
new file mode 100644
index 0000000..78b5315
--- /dev/null
+++ b/internal/cli/winch_unix.go
@@ -0,0 +1,41 @@
+//go:build !windows
+
+package cli
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"golang.org/x/term"
+
+	"pxmon/internal/cluster"
+)
+
+func installWinchHandler(fd int, resize chan<- cluster.InteractiveShellSize) chan os.Signal {
+	sigCh := make(chan os.Signal, 4)
+	signal.Notify(sigCh, syscall.SIGWINCH)
+	go func() {
+		for range sigCh {
+			w, h, err := term.GetSize(fd)
+			if err != nil || w <= 0 || h <= 0 {
+				continue
+			}
+			select {
+			case resize <- cluster.InteractiveShellSize{Width: w, Height: h}:
+			default:
+			}
+		}
+	}()
+	return sigCh
+}
+
+func closeWinchHandler(sigCh chan os.Signal, resize chan cluster.InteractiveShellSize) {
+	if sigCh != nil {
+		signal.Stop(sigCh)
+		close(sigCh)
+	}
+	if resize != nil {
+		close(resize)
+	}
+}
diff --git a/internal/cli/winch_windows.go b/internal/cli/winch_windows.go
new file mode 100644
index 0000000..3a41f75
--- /dev/null
+++ b/internal/cli/winch_windows.go
@@ -0,0 +1,19 @@
+//go:build windows
+
+package cli
+
+import (
+	"os"
+
+	"pxmon/internal/cluster"
+)
+
+func installWinchHandler(fd int, resize chan<- cluster.InteractiveShellSize) chan os.Signal {
+	return nil
+}
+
+func closeWinchHandler(sigCh chan os.Signal, resize chan cluster.InteractiveShellSize) {
+	if resize != nil {
+		close(resize)
+	}
+}
diff --git a/internal/cluster/agent_auth.go b/internal/cluster/agent_auth.go
new file mode 100644
index 0000000..8e8016c
--- /dev/null
+++ b/internal/cluster/agent_auth.go
@@ -0,0 +1,42 @@
+package cluster
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	agentHeaderTS        = "X-Observer-Ts"
+	agentHeaderNonce     = "X-Observer-Nonce"
+	agentHeaderSignature = "X-Observer-Signature"
+)
+
+func applyAgentRequestAuth(req *http.Request, c Cluster) {
+	if req == nil {
+		return
+	}
+	if strings.TrimSpace(c.Agent.Token) != "" {
+		req.Header.Set("Authorization", "Bearer "+c.Agent.Token)
+	}
+	secret := strings.TrimSpace(c.Agent.RequestSecret)
+	if secret == "" {
+		return
+	}
+
+	ts := strconv.FormatInt(time.Now().UTC().Unix(), 10)
+	nonce := randomHex(12)
+	payload := req.Method + "\n" + req.URL.RequestURI() + "\n" + ts + "\n" + nonce
+
+	mac := hmac.New(sha256.New, []byte(secret))
+	_, _ = mac.Write([]byte(payload))
+	sig := hex.EncodeToString(mac.Sum(nil))
+
+	req.Header.Set(agentHeaderTS, ts)
+	req.Header.Set(agentHeaderNonce, nonce)
+	req.Header.Set(agentHeaderSignature, sig)
+}
diff --git a/internal/cluster/agent_versions.go b/internal/cluster/agent_versions.go
new file mode 100644
index 0000000..6af1fbc
--- /dev/null
+++ b/internal/cluster/agent_versions.go
@@ -0,0 +1,68 @@
+package cluster
+
+import (
+	"embed"
+	"encoding/json"
+	"strings"
+)
+
+//go:embed agent_versions.json
+var agentVersionsFS embed.FS
+
+type AgentVersionInfo struct {
+	Version    string   `json:"version"`
+	ReleasedAt string   `json:"released_at,omitempty"`
+	Features   []string `json:"features,omitempty"`
+}
+
+func loadAgentVersions() []AgentVersionInfo {
+	raw, err := agentVersionsFS.ReadFile("agent_versions.json")
+	if err != nil {
+		return nil
+	}
+	var items []AgentVersionInfo
+	if err := json.Unmarshal(raw, &items); err != nil {
+		return nil
+	}
+	out := make([]AgentVersionInfo, 0, len(items))
+	for _, it := range items {
+		it.Version = strings.TrimSpace(it.Version)
+		if it.Version == "" {
+			continue
+		}
+		out = append(out, it)
+	}
+	return out
+}
+
+func (s *Service) AgentVersions() []AgentVersionInfo {
+	return append([]AgentVersionInfo(nil), loadAgentVersions()...)
+}
+
+func (s *Service) AgentVersionFeatures(version string) []string {
+	v := strings.TrimSpace(version)
+	if v == "" {
+		return nil
+	}
+	for _, it := range loadAgentVersions() {
+		if strings.EqualFold(strings.TrimSpace(it.Version), v) {
+			return append([]string(nil), it.Features...)
+		}
+	}
+	return nil
+}
+
+func (s *Service) CompareAgentVersion(version string) (isLatest bool, latest string, known bool) {
+	items := loadAgentVersions()
+	if len(items) == 0 {
+		return true, "", false
+	}
+	latest = strings.TrimSpace(items[len(items)-1].Version)
+	v := strings.TrimSpace(version)
+	for _, it := range items {
+		if strings.EqualFold(strings.TrimSpace(it.Version), v) {
+			return strings.EqualFold(v, latest), latest, true
+		}
+	}
+	return false, latest, false
+}
diff --git a/internal/cluster/agent_versions.json b/internal/cluster/agent_versions.json
new file mode 100644
index 0000000..0ee29ce
--- /dev/null
+++ b/internal/cluster/agent_versions.json
@@ -0,0 +1,37 @@
+[
+  {
+    "version": "dev",
+    "released_at": "2026-04-10",
+    "features": [
+      "cluster usage/traffic/graph",
+      "kvm top static spec",
+      "telegram graph sendPhoto fallback"
+    ]
+  },
+  {
+    "version": "dev-2026.04.17",
+    "released_at": "2026-04-17",
+    "features": [
+      "cluster p95 by interface",
+      "cluster tag + kvm tag",
+      "cluster vm alert-rules",
+      "cluster drift",
+      "cluster runbook",
+      "cluster scheduler",
+      "cluster change-history",
+      "cluster report export"
+    ]
+  },
+  {
+    "version": "v0.2.0",
+    "released_at": "2026-06-16",
+    "features": [
+      "repo tunneling gateway/proxy workflows",
+      "cluster exec/run commands",
+      "ssh key passphrase file support",
+      "export/import referenced key files",
+      "signed agent request skew hardening",
+      "tui refresh and network dashboard improvements"
+    ]
+  }
+]
diff --git a/internal/cluster/backup.go b/internal/cluster/backup.go
new file mode 100644
index 0000000..c918754
--- /dev/null
+++ b/internal/cluster/backup.go
@@ -0,0 +1,655 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+type BackupRunResult struct {
+	PlanID      string    `json:"plan_id"`
+	PlanName    string    `json:"plan_name"`
+	TargetID    string    `json:"target_id"`
+	TargetName  string    `json:"target_name"`
+	ArchiveName string    `json:"archive_name"`
+	UploadedTo  string    `json:"uploaded_to"`
+	SizeBytes   int64     `json:"size_bytes"`
+	RanAt       time.Time `json:"ran_at"`
+}
+
+func (s *Service) BackupListTargets() ([]BackupTarget, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return nil, err
+	}
+	return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil
+}
+
+func (s *Service) BackupListPlans() ([]BackupPlan, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return nil, err
+	}
+	return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil
+}
+
+func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return "", err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+	selector = strings.TrimSpace(selector)
+	var target *BackupTarget
+	for i := range cfg.Targets {
+		if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) {
+			target = &cfg.Targets[i]
+			break
+		}
+	}
+	if target == nil {
+		return "", fmt.Errorf("backup target not found")
+	}
+	switch target.Type {
+	case "sftp":
+		host := strings.TrimSpace(target.SFTPHost)
+		user := strings.TrimSpace(target.SFTPUser)
+		if host == "" || user == "" {
+			return "", fmt.Errorf("sftp target has empty host/user")
+		}
+		addr := netJoinHostPort(host, target.SFTPPort)
+		auths := make([]ssh.AuthMethod, 0, 2)
+		if strings.TrimSpace(target.SFTPPassword) != "" {
+			auths = append(auths, ssh.Password(target.SFTPPassword))
+		}
+		if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" {
+			pemBytes, err := os.ReadFile(kp)
+			if err != nil {
+				return "", fmt.Errorf("read sftp key: %w", err)
+			}
+			signer, err := ssh.ParsePrivateKey(pemBytes)
+			if err != nil {
+				return "", fmt.Errorf("parse sftp key: %w", err)
+			}
+			auths = append(auths, ssh.PublicKeys(signer))
+		}
+		if len(auths) == 0 {
+			return "", fmt.Errorf("sftp auth is required (password or key)")
+		}
+		sshCfg := &ssh.ClientConfig{
+			User:            user,
+			Auth:            auths,
+			HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+			Timeout:         15 * time.Second,
+		}
+		conn, err := ssh.Dial("tcp", addr, sshCfg)
+		if err != nil {
+			return "", err
+		}
+		defer conn.Close()
+		c, err := sftp.NewClient(conn)
+		if err != nil {
+			return "", err
+		}
+		defer c.Close()
+		base := strings.TrimSpace(target.SFTPBasePath)
+		if base == "" {
+			base = "."
+		}
+		if err := c.MkdirAll(base); err != nil {
+			return "", err
+		}
+		if _, err := c.ReadDir(base); err != nil {
+			return "", err
+		}
+		return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil
+	case "s3":
+		endpoint := strings.TrimSpace(target.S3Endpoint)
+		bucket := strings.TrimSpace(target.S3Bucket)
+		access := strings.TrimSpace(target.S3AccessKey)
+		secret := strings.TrimSpace(target.S3SecretKey)
+		if endpoint == "" || bucket == "" || access == "" || secret == "" {
+			return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
+		}
+		region := strings.TrimSpace(target.S3Region)
+		if region == "" {
+			region = "us-east-1"
+		}
+		lookup := minio.BucketLookupAuto
+		if target.S3PathStyle {
+			lookup = minio.BucketLookupPath
+		}
+		cli, err := minio.New(endpoint, &minio.Options{
+			Creds:        credentials.NewStaticV4(access, secret, ""),
+			Secure:       target.S3UseSSL,
+			Region:       region,
+			BucketLookup: lookup,
+		})
+		if err != nil {
+			return "", err
+		}
+		exists, err := cli.BucketExists(ctx, bucket)
+		if err != nil {
+			return "", err
+		}
+		if !exists {
+			return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket)
+		}
+		scheme := "https"
+		if !target.S3UseSSL {
+			scheme = "http"
+		}
+		return scheme + "://" + endpoint + "/" + bucket, nil
+	default:
+		return "", fmt.Errorf("unsupported target type %q", target.Type)
+	}
+}
+
+func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return BackupTarget{}, err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+	t.ID = strings.TrimSpace(t.ID)
+	if t.ID == "" {
+		t.ID = newClusterID()
+	}
+	t.Name = strings.TrimSpace(t.Name)
+	if t.Name == "" {
+		return BackupTarget{}, fmt.Errorf("target name is required")
+	}
+	t.Type = strings.ToLower(strings.TrimSpace(t.Type))
+	if t.Type != "sftp" && t.Type != "s3" {
+		return BackupTarget{}, fmt.Errorf("target type must be sftp|s3")
+	}
+	if t.Type == "sftp" {
+		if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" {
+			return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user")
+		}
+		if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" {
+			return BackupTarget{}, fmt.Errorf("sftp target requires password or key")
+		}
+	}
+	if t.Type == "s3" {
+		if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" {
+			return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket")
+		}
+		if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" {
+			return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key")
+		}
+	}
+	for _, ex := range cfg.Targets {
+		if strings.EqualFold(ex.Name, t.Name) {
+			return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name)
+		}
+	}
+	now := s.now().UTC()
+	t.CreatedAt = now
+	t.UpdatedAt = now
+	if t.SFTPPort <= 0 {
+		t.SFTPPort = 22
+	}
+	if !t.Enabled {
+		t.Enabled = true
+	}
+	cfg.Targets = append(cfg.Targets, t)
+	reg.Backups = cfg
+	if err := s.store.Save(reg); err != nil {
+		return BackupTarget{}, err
+	}
+	_ = s.AppendChange("backup.target.add", t.ID, t.Name)
+	return t, nil
+}
+
+func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return BackupTarget{}, err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+	selector = strings.TrimSpace(selector)
+	idx := -1
+	for i, t := range cfg.Targets {
+		if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
+			idx = i
+			break
+		}
+	}
+	if idx < 0 {
+		return BackupTarget{}, fmt.Errorf("backup target not found")
+	}
+	removed := cfg.Targets[idx]
+	cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...)
+	reg.Backups = cfg
+	if err := s.store.Save(reg); err != nil {
+		return BackupTarget{}, err
+	}
+	_ = s.AppendChange("backup.target.remove", removed.ID, removed.Name)
+	return removed, nil
+}
+
+func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return BackupPlan{}, err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+	if strings.TrimSpace(p.Name) == "" {
+		return BackupPlan{}, fmt.Errorf("plan name is required")
+	}
+	if strings.TrimSpace(p.TargetID) == "" {
+		return BackupPlan{}, fmt.Errorf("target is required")
+	}
+	if len(p.Paths) == 0 {
+		return BackupPlan{}, fmt.Errorf("at least one path is required")
+	}
+	targetID := ""
+	for _, t := range cfg.Targets {
+		if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) {
+			targetID = t.ID
+			break
+		}
+	}
+	if targetID == "" {
+		return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID)
+	}
+	for _, ex := range cfg.Plans {
+		if strings.EqualFold(ex.Name, p.Name) {
+			return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name)
+		}
+	}
+	p.ID = newClusterID()
+	p.TargetID = targetID
+	p.Paths = normalizeBackupPaths(p.Paths)
+	if strings.TrimSpace(p.Every) == "" {
+		p.Every = "24h"
+	}
+	if p.RetainDays <= 0 {
+		p.RetainDays = 30
+	}
+	p.Compress = true
+	now := s.now().UTC()
+	p.CreatedAt = now
+	p.UpdatedAt = now
+	if !p.Enabled {
+		p.Enabled = true
+	}
+	cfg.Plans = append(cfg.Plans, p)
+	reg.Backups = cfg
+	if err := s.store.Save(reg); err != nil {
+		return BackupPlan{}, err
+	}
+	_ = s.AppendChange("backup.plan.add", p.ID, p.Name)
+	return p, nil
+}
+
+func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return BackupPlan{}, err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+	selector = strings.TrimSpace(selector)
+	idx := -1
+	for i, p := range cfg.Plans {
+		if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) {
+			idx = i
+			break
+		}
+	}
+	if idx < 0 {
+		return BackupPlan{}, fmt.Errorf("backup plan not found")
+	}
+	removed := cfg.Plans[idx]
+	cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
+	reg.Backups = cfg
+	if err := s.store.Save(reg); err != nil {
+		return BackupPlan{}, err
+	}
+	_ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name)
+	return removed, nil
+}
+
+func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return BackupRunResult{}, err
+	}
+	cfg := normalizeBackupConfig(reg.Backups)
+
+	var plan *BackupPlan
+	for i := range cfg.Plans {
+		if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) {
+			plan = &cfg.Plans[i]
+			break
+		}
+	}
+	if plan == nil {
+		return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector)
+	}
+	var target *BackupTarget
+	for i := range cfg.Targets {
+		if cfg.Targets[i].ID == plan.TargetID {
+			target = &cfg.Targets[i]
+			break
+		}
+	}
+	if target == nil {
+		return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID)
+	}
+
+	c, err := s.Get(plan.Cluster)
+	if err != nil {
+		return BackupRunResult{}, err
+	}
+	sshClient, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err)
+	}
+	defer sshClient.Close()
+
+	ts := s.now().UTC().Format("20060102T150405Z")
+	archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz"
+	tmpPath := filepath.Join(os.TempDir(), archiveName)
+	tmpFile, err := os.Create(tmpPath)
+	if err != nil {
+		return BackupRunResult{}, err
+	}
+	defer func() {
+		_ = tmpFile.Close()
+		_ = os.Remove(tmpPath)
+	}()
+
+	remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths)
+	if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil {
+		plan.LastRunAt = s.now().UTC()
+		plan.LastStatus = "failed"
+		plan.LastError = err.Error()
+		plan.UpdatedAt = s.now().UTC()
+		reg.Backups = cfg
+		_ = s.store.Save(reg)
+		return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err)
+	}
+	if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
+		return BackupRunResult{}, err
+	}
+	st, _ := tmpFile.Stat()
+	size := int64(0)
+	if st != nil {
+		size = st.Size()
+	}
+
+	uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size)
+	if err != nil {
+		plan.LastRunAt = s.now().UTC()
+		plan.LastStatus = "failed"
+		plan.LastError = err.Error()
+		plan.UpdatedAt = s.now().UTC()
+		reg.Backups = cfg
+		_ = s.store.Save(reg)
+		return BackupRunResult{}, fmt.Errorf("upload backup: %w", err)
+	}
+
+	plan.LastRunAt = s.now().UTC()
+	plan.LastStatus = "ok"
+	plan.LastError = ""
+	plan.LastArchive = archiveName
+	plan.UpdatedAt = s.now().UTC()
+	reg.Backups = cfg
+	if err := s.store.Save(reg); err != nil {
+		return BackupRunResult{}, err
+	}
+	_ = s.AppendChange("backup.plan.run", plan.ID, archiveName)
+
+	return BackupRunResult{
+		PlanID:      plan.ID,
+		PlanName:    plan.Name,
+		TargetID:    target.ID,
+		TargetName:  target.Name,
+		ArchiveName: archiveName,
+		UploadedTo:  uploadedTo,
+		SizeBytes:   size,
+		RanAt:       plan.LastRunAt,
+	}, nil
+}
+
+func normalizeBackupPaths(in []string) []string {
+	out := make([]string, 0, len(in))
+	seen := map[string]struct{}{}
+	for _, p := range in {
+		v := strings.TrimSpace(p)
+		if v == "" {
+			continue
+		}
+		if _, ok := seen[v]; ok {
+			continue
+		}
+		seen[v] = struct{}{}
+		out = append(out, v)
+	}
+	return out
+}
+
+func sanitizeBackupName(v string) string {
+	v = strings.ToLower(strings.TrimSpace(v))
+	if v == "" {
+		return "backup"
+	}
+	var b strings.Builder
+	for _, r := range v {
+		switch {
+		case r >= 'a' && r <= 'z':
+			b.WriteRune(r)
+		case r >= '0' && r <= '9':
+			b.WriteRune(r)
+		case r == '-' || r == '_' || r == '.':
+			b.WriteRune(r)
+		default:
+			b.WriteByte('-')
+		}
+	}
+	out := strings.Trim(b.String(), "-")
+	if out == "" {
+		return "backup"
+	}
+	return out
+}
+
+func buildRemoteTarStreamCommand(paths []string) string {
+	items := make([]string, 0, len(paths))
+	for _, p := range paths {
+		v := strings.TrimSpace(p)
+		if v == "" {
+			continue
+		}
+		items = append(items, shellQuote(v))
+	}
+	if len(items) == 0 {
+		items = []string{shellQuote("/")}
+	}
+	return "tar -czf - " + strings.Join(items, " ")
+}
+
+func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error {
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+	stdout, err := session.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	stderr, err := session.StderrPipe()
+	if err != nil {
+		return err
+	}
+	if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
+		return err
+	}
+	done := make(chan error, 1)
+	go func() {
+		_, cpErr := io.Copy(w, stdout)
+		if cpErr != nil {
+			done <- cpErr
+			return
+		}
+		done <- session.Wait()
+	}()
+	select {
+	case <-ctx.Done():
+		_ = session.Close()
+		return ctx.Err()
+	case err := <-done:
+		if err == nil {
+			return nil
+		}
+		b, _ := io.ReadAll(stderr)
+		msg := strings.TrimSpace(string(b))
+		if msg == "" {
+			return err
+		}
+		return fmt.Errorf("%w: %s", err, msg)
+	}
+}
+
+func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(target.Type)) {
+	case "sftp":
+		return uploadBackupSFTP(ctx, target, archiveName, r)
+	case "s3":
+		return uploadBackupS3(ctx, target, archiveName, r, size)
+	default:
+		return "", fmt.Errorf("unsupported backup target type %q", target.Type)
+	}
+}
+
+func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) {
+	_ = ctx
+	host := strings.TrimSpace(t.SFTPHost)
+	if host == "" {
+		return "", fmt.Errorf("sftp_host is required")
+	}
+	user := strings.TrimSpace(t.SFTPUser)
+	if user == "" {
+		return "", fmt.Errorf("sftp_user is required")
+	}
+	addr := netJoinHostPort(host, t.SFTPPort)
+	auths := make([]ssh.AuthMethod, 0, 2)
+	if strings.TrimSpace(t.SFTPPassword) != "" {
+		auths = append(auths, ssh.Password(t.SFTPPassword))
+	}
+	if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" {
+		pemBytes, err := os.ReadFile(kp)
+		if err != nil {
+			return "", fmt.Errorf("read sftp key: %w", err)
+		}
+		signer, err := ssh.ParsePrivateKey(pemBytes)
+		if err != nil {
+			return "", fmt.Errorf("parse sftp key: %w", err)
+		}
+		auths = append(auths, ssh.PublicKeys(signer))
+	}
+	if len(auths) == 0 {
+		return "", fmt.Errorf("sftp auth is required (password or key)")
+	}
+	sshCfg := &ssh.ClientConfig{
+		User:            user,
+		Auth:            auths,
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust
+		Timeout:         15 * time.Second,
+	}
+	conn, err := ssh.Dial("tcp", addr, sshCfg)
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+	c, err := sftp.NewClient(conn)
+	if err != nil {
+		return "", err
+	}
+	defer c.Close()
+
+	base := strings.TrimSpace(t.SFTPBasePath)
+	if base == "" {
+		base = "."
+	}
+	if err := c.MkdirAll(base); err != nil {
+		return "", err
+	}
+	remote := path.Join(base, archiveName)
+	f, err := c.Create(remote)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	if _, err := io.Copy(f, r); err != nil {
+		return "", err
+	}
+	return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil
+}
+
+func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
+	endpoint := strings.TrimSpace(t.S3Endpoint)
+	bucket := strings.TrimSpace(t.S3Bucket)
+	access := strings.TrimSpace(t.S3AccessKey)
+	secret := strings.TrimSpace(t.S3SecretKey)
+	if endpoint == "" || bucket == "" || access == "" || secret == "" {
+		return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
+	}
+	region := strings.TrimSpace(t.S3Region)
+	if region == "" {
+		region = "us-east-1"
+	}
+	lookup := minio.BucketLookupAuto
+	if t.S3PathStyle {
+		lookup = minio.BucketLookupPath
+	}
+	cli, err := minio.New(endpoint, &minio.Options{
+		Creds:        credentials.NewStaticV4(access, secret, ""),
+		Secure:       t.S3UseSSL,
+		Region:       region,
+		BucketLookup: lookup,
+	})
+	if err != nil {
+		return "", err
+	}
+	key := archiveName
+	if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" {
+		key = p + "/" + archiveName
+	}
+	opts := minio.PutObjectOptions{ContentType: "application/gzip"}
+	if size < 0 {
+		size = -1
+	}
+	_, err = cli.PutObject(ctx, bucket, key, r, size, opts)
+	if err != nil {
+		return "", err
+	}
+	scheme := "https"
+	if !t.S3UseSSL {
+		scheme = "http"
+	}
+	return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil
+}
+
+func netJoinHostPort(host string, port int) string {
+	p := port
+	if p <= 0 {
+		p = 22
+	}
+	return net.JoinHostPort(host, strconv.Itoa(p))
+}
diff --git a/internal/cluster/change_history.go b/internal/cluster/change_history.go
new file mode 100644
index 0000000..35cc256
--- /dev/null
+++ b/internal/cluster/change_history.go
@@ -0,0 +1,89 @@
+package cluster
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type ChangeRecord struct {
+	At      time.Time `json:"at"`
+	Action  string    `json:"action"`
+	Target  string    `json:"target"`
+	Details string    `json:"details,omitempty"`
+}
+
+func (s *Service) changeHistoryPath() string {
+	return filepath.Join(s.DataDir(), "history", "changes.log")
+}
+
+func (s *Service) AppendChange(action, target, details string) error {
+	if s == nil {
+		return nil
+	}
+	rec := ChangeRecord{
+		At:      s.now().UTC(),
+		Action:  strings.TrimSpace(action),
+		Target:  strings.TrimSpace(target),
+		Details: strings.TrimSpace(details),
+	}
+	if rec.Action == "" {
+		return nil
+	}
+	path := s.changeHistoryPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	b, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(append(b, '\n'))
+	return err
+}
+
+func (s *Service) ListChanges(limit int) ([]ChangeRecord, error) {
+	path := s.changeHistoryPath()
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	capHint := 32
+	if limit > capHint {
+		capHint = limit
+	}
+	items := make([]ChangeRecord, 0, capHint)
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var rec ChangeRecord
+		if err := json.Unmarshal([]byte(line), &rec); err != nil {
+			continue
+		}
+		items = append(items, rec)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("scan change history: %w", err)
+	}
+	if limit > 0 && len(items) > limit {
+		items = items[len(items)-limit:]
+	}
+	return items, nil
+}
diff --git a/internal/cluster/drift.go b/internal/cluster/drift.go
new file mode 100644
index 0000000..0e70e95
--- /dev/null
+++ b/internal/cluster/drift.go
@@ -0,0 +1,86 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type DriftIssue struct {
+	Level   string `json:"level"`
+	Kind    string `json:"kind"`
+	Message string `json:"message"`
+}
+
+type DriftReport struct {
+	Cluster   string       `json:"cluster"`
+	Generated time.Time    `json:"generated_at"`
+	Issues    []DriftIssue `json:"issues,omitempty"`
+}
+
+func (s *Service) DetectDrift(ctx context.Context, selector string) (DriftReport, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return DriftReport{}, err
+	}
+	rep := DriftReport{Cluster: c.Name, Generated: s.now().UTC()}
+	expected := strings.TrimSpace(s.ExpectedAgentVersion())
+	dctrl := normalizeDriftControl(c.Drift)
+
+	if !c.Agent.Installed {
+		rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent", Message: "agent is not installed"})
+		return rep, nil
+	}
+
+	pingCtx, cancel := context.WithTimeout(ctx, 2500*time.Millisecond)
+	ping, pingErr := s.PingAgent(pingCtx, c.ID)
+	cancel()
+	if pingErr != nil || !ping.Reachable || ping.StatusCode >= 400 {
+		rep.Issues = append(rep.Issues, DriftIssue{Level: "crit", Kind: "agent", Message: "agent is unreachable"})
+	} else {
+		nodeVersion := strings.TrimSpace(ping.Version)
+		if nodeVersion == "" {
+			nodeVersion = strings.TrimSpace(c.Agent.Version)
+		}
+		if expected != "" && nodeVersion != "" && nodeVersion != expected {
+			rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_version", Message: fmt.Sprintf("agent version mismatch: node=%s local=%s", nodeVersion, expected)})
+		}
+		if nodeVersion != "" {
+			if ok, latest, known := s.CompareAgentVersion(nodeVersion); known && !ok {
+				rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_outdated", Message: fmt.Sprintf("node version %s is older than latest known %s", nodeVersion, latest)})
+			}
+			if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.AgentVersion) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.AgentVersion), nodeVersion) {
+				rep.Issues = append(rep.Issues, DriftIssue{
+					Level:   "warn",
+					Kind:    "baseline_agent_version",
+					Message: fmt.Sprintf("baseline agent version mismatch: baseline=%s live=%s", dctrl.Baseline.AgentVersion, nodeVersion),
+				})
+			}
+		}
+	}
+
+	softCtx, softCancel := context.WithTimeout(ctx, 7*time.Second)
+	fresh, softErr := s.probeSoftware(softCtx, c, "", "")
+	softCancel()
+	if softErr != nil {
+		rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software_probe", Message: softErr.Error()})
+	} else {
+		oldSet := strings.TrimSpace(c.Software.Summary())
+		newSet := strings.TrimSpace(fresh.Summary())
+		if oldSet != "" && oldSet != "-" && newSet != oldSet {
+			rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software", Message: fmt.Sprintf("software support changed: stored=%s live=%s", oldSet, newSet)})
+		}
+		if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.Software) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.Software), newSet) {
+			rep.Issues = append(rep.Issues, DriftIssue{
+				Level:   "warn",
+				Kind:    "baseline_software",
+				Message: fmt.Sprintf("baseline software mismatch: baseline=%s live=%s", dctrl.Baseline.Software, newSet),
+			})
+		}
+	}
+
+	sort.Slice(rep.Issues, func(i, j int) bool { return rep.Issues[i].Kind < rep.Issues[j].Kind })
+	return rep, nil
+}
diff --git a/internal/cluster/export.go b/internal/cluster/export.go
new file mode 100644
index 0000000..af3d093
--- /dev/null
+++ b/internal/cluster/export.go
@@ -0,0 +1,456 @@
+package cluster
+
+import (
+	"crypto/aes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/scrypt"
+)
+
+const (
+	exportMagic   = "OBSCTLEXP1:"
+	exportVersion = 2
+	saltBytes     = 16
+	scryptN       = 1 << 15
+	scryptR       = 8
+	scryptP       = 1
+)
+
+// ExportBundle is the portable payload produced by Service.Export. It is
+// self-contained: it includes every cluster (with stored credentials),
+// telegram bot settings, locker config, and alert policies.
+type ExportBundle struct {
+	Version    int            `json:"version"`
+	ExportedAt string         `json:"exported_at"`
+	Registry   Registry       `json:"registry"`
+	Files      []ExportedFile `json:"files,omitempty"`
+}
+
+type ExportedFile struct {
+	OriginalPath string `json:"original_path"`
+	Kind         string `json:"kind"`
+	Content      []byte `json:"content"`
+}
+
+// ImportMode controls how an imported bundle is merged into the current store.
+type ImportMode string
+
+const (
+	// ImportModeMerge adds clusters from the bundle; existing clusters with
+	// the same name are replaced with the imported copy.
+	ImportModeMerge ImportMode = "merge"
+	// ImportModeReplace wipes the current registry and replaces it with the
+	// imported bundle as-is.
+	ImportModeReplace ImportMode = "replace"
+)
+
+// ImportReport summarizes what happened during an import.
+type ImportReport struct {
+	Added           int
+	Replaced        int
+	TotalAfter      int
+	TelegramApplied bool
+	LockerApplied   bool
+	Mode            ImportMode
+}
+
+// Export writes an encrypted, passphrase-protected bundle of the full
+// registry to outPath. The bundle can be imported on another machine with
+// the same password — no master key transfer required.
+func (s *Service) Export(outPath, password string) error {
+	if strings.TrimSpace(outPath) == "" {
+		return errors.New("export: output path is required")
+	}
+	if strings.TrimSpace(password) == "" {
+		return errors.New("export: password is required")
+	}
+
+	reg, err := s.store.Load()
+	if err != nil {
+		return fmt.Errorf("export: load registry: %w", err)
+	}
+
+	files, err := collectExportFiles(reg)
+	if err != nil {
+		return err
+	}
+
+	bundle := ExportBundle{
+		Version:    exportVersion,
+		ExportedAt: s.now().UTC().Format(time.RFC3339),
+		Registry:   reg,
+		Files:      files,
+	}
+
+	payload, err := json.MarshalIndent(bundle, "", "  ")
+	if err != nil {
+		return fmt.Errorf("export: encode bundle: %w", err)
+	}
+
+	salt := make([]byte, saltBytes)
+	if _, err := rand.Read(salt); err != nil {
+		return fmt.Errorf("export: generate salt: %w", err)
+	}
+
+	key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
+	if err != nil {
+		return fmt.Errorf("export: derive key: %w", err)
+	}
+
+	ciphertext, err := encrypt(payload, key)
+	if err != nil {
+		return fmt.Errorf("export: encrypt: %w", err)
+	}
+
+	blob := append([]byte{}, salt...)
+	blob = append(blob, ciphertext...)
+	encoded := exportMagic + base64.StdEncoding.EncodeToString(blob) + "\n"
+
+	if dir := filepath.Dir(outPath); dir != "" && dir != "." {
+		if err := os.MkdirAll(dir, 0o700); err != nil {
+			return fmt.Errorf("export: create output dir: %w", err)
+		}
+	}
+
+	tmp := outPath + ".tmp"
+	if err := os.WriteFile(tmp, []byte(encoded), 0o600); err != nil {
+		return fmt.Errorf("export: write temp: %w", err)
+	}
+	if err := os.Rename(tmp, outPath); err != nil {
+		return fmt.Errorf("export: replace output: %w", err)
+	}
+	return nil
+}
+
+// Import reads an exported bundle from inPath using password, and applies
+// it to the local registry according to mode.
+func (s *Service) Import(inPath, password string, mode ImportMode) (ImportReport, error) {
+	if strings.TrimSpace(inPath) == "" {
+		return ImportReport{}, errors.New("import: input path is required")
+	}
+	if strings.TrimSpace(password) == "" {
+		return ImportReport{}, errors.New("import: password is required")
+	}
+	if mode == "" {
+		mode = ImportModeMerge
+	}
+	if mode != ImportModeMerge && mode != ImportModeReplace {
+		return ImportReport{}, fmt.Errorf("import: unsupported mode %q", mode)
+	}
+
+	raw, err := os.ReadFile(inPath)
+	if err != nil {
+		return ImportReport{}, fmt.Errorf("import: read input: %w", err)
+	}
+	text := strings.TrimSpace(string(raw))
+	if !strings.HasPrefix(text, exportMagic) {
+		return ImportReport{}, errors.New("import: not a PXmon export bundle")
+	}
+	blob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(text, exportMagic))
+	if err != nil {
+		return ImportReport{}, fmt.Errorf("import: decode payload: %w", err)
+	}
+	if len(blob) < saltBytes+aes.BlockSize {
+		return ImportReport{}, errors.New("import: payload truncated")
+	}
+	salt := blob[:saltBytes]
+	ciphertext := blob[saltBytes:]
+
+	key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
+	if err != nil {
+		return ImportReport{}, fmt.Errorf("import: derive key: %w", err)
+	}
+
+	plaintext, err := decrypt(ciphertext, key)
+	if err != nil {
+		return ImportReport{}, errors.New("import: wrong password or corrupt bundle")
+	}
+
+	var bundle ExportBundle
+	if err := json.Unmarshal(plaintext, &bundle); err != nil {
+		return ImportReport{}, fmt.Errorf("import: parse bundle: %w", err)
+	}
+	if bundle.Version == 0 || bundle.Version > exportVersion {
+		return ImportReport{}, fmt.Errorf("import: unsupported bundle version %d", bundle.Version)
+	}
+
+	incoming := bundle.Registry
+	if incoming.Clusters == nil {
+		incoming.Clusters = []Cluster{}
+	}
+	if len(bundle.Files) > 0 {
+		restored, restoreErr := s.restoreExportFiles(bundle.Files)
+		if restoreErr != nil {
+			return ImportReport{}, restoreErr
+		}
+		applyRestoredFilePaths(&incoming, restored)
+	}
+
+	report := ImportReport{Mode: mode}
+
+	if mode == ImportModeReplace {
+		report.Added = len(incoming.Clusters)
+		report.TotalAfter = len(incoming.Clusters)
+		report.TelegramApplied = incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0
+		report.LockerApplied = incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled
+		if err := s.store.Save(incoming); err != nil {
+			return ImportReport{}, fmt.Errorf("import: save: %w", err)
+		}
+		return report, nil
+	}
+
+	current, err := s.store.Load()
+	if err != nil {
+		return ImportReport{}, fmt.Errorf("import: load current: %w", err)
+	}
+	if current.Clusters == nil {
+		current.Clusters = []Cluster{}
+	}
+
+	existingByName := make(map[string]int, len(current.Clusters))
+	for i, c := range current.Clusters {
+		existingByName[strings.ToLower(c.Name)] = i
+	}
+
+	for _, inc := range incoming.Clusters {
+		key := strings.ToLower(strings.TrimSpace(inc.Name))
+		if key == "" {
+			continue
+		}
+		inc.Alerts = ensureAlertPolicy(inc.Alerts)
+		if idx, ok := existingByName[key]; ok {
+			// Preserve original ID to keep references stable.
+			inc.ID = current.Clusters[idx].ID
+			current.Clusters[idx] = inc
+			report.Replaced++
+		} else {
+			if strings.TrimSpace(inc.ID) == "" {
+				inc.ID = newClusterID()
+			}
+			current.Clusters = append(current.Clusters, inc)
+			existingByName[key] = len(current.Clusters) - 1
+			report.Added++
+		}
+	}
+
+	if incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0 {
+		current.Telegram = incoming.Telegram
+		report.TelegramApplied = true
+	}
+	if incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled {
+		current.Locker = incoming.Locker
+		report.LockerApplied = true
+	}
+	current.Backups = mergeBackupConfig(current.Backups, incoming.Backups)
+	if strings.TrimSpace(current.ActiveClusterID) == "" {
+		current.ActiveClusterID = incoming.ActiveClusterID
+	}
+
+	report.TotalAfter = len(current.Clusters)
+	if err := s.store.Save(current); err != nil {
+		return ImportReport{}, fmt.Errorf("import: save: %w", err)
+	}
+	return report, nil
+}
+
+func collectExportFiles(reg Registry) ([]ExportedFile, error) {
+	type wantFile struct {
+		path string
+		kind string
+	}
+	wants := make([]wantFile, 0)
+	for _, c := range reg.Clusters {
+		if strings.TrimSpace(c.KeyPath) != "" {
+			wants = append(wants, wantFile{path: c.KeyPath, kind: "ssh_private_key"})
+		}
+		if strings.TrimSpace(c.KeyPassphraseFile) != "" {
+			wants = append(wants, wantFile{path: c.KeyPassphraseFile, kind: "ssh_key_passphrase"})
+		}
+	}
+	for _, t := range reg.Backups.Targets {
+		if strings.TrimSpace(t.SFTPKeyPath) != "" {
+			wants = append(wants, wantFile{path: t.SFTPKeyPath, kind: "sftp_private_key"})
+		}
+	}
+
+	seen := map[string]struct{}{}
+	files := make([]ExportedFile, 0, len(wants))
+	for _, w := range wants {
+		expanded, err := expandPath(w.path)
+		if err != nil {
+			return nil, fmt.Errorf("export: resolve %s file %q: %w", w.kind, w.path, err)
+		}
+		if _, ok := seen[expanded]; ok {
+			continue
+		}
+		data, err := os.ReadFile(expanded)
+		if err != nil {
+			return nil, fmt.Errorf("export: read %s file %q: %w", w.kind, expanded, err)
+		}
+		seen[expanded] = struct{}{}
+		files = append(files, ExportedFile{
+			OriginalPath: expanded,
+			Kind:         w.kind,
+			Content:      data,
+		})
+	}
+	return files, nil
+}
+
+func (s *Service) restoreExportFiles(files []ExportedFile) (map[string]string, error) {
+	base := filepath.Join(filepath.Dir(s.store.Path()), "imported-files")
+	if err := os.MkdirAll(base, 0o700); err != nil {
+		return nil, fmt.Errorf("import: create imported-files dir: %w", err)
+	}
+
+	restored := make(map[string]string, len(files))
+	for _, f := range files {
+		orig := strings.TrimSpace(f.OriginalPath)
+		if orig == "" {
+			continue
+		}
+		name := filepath.Base(orig)
+		if name == "." || name == string(filepath.Separator) || strings.TrimSpace(name) == "" {
+			name = "secret"
+		}
+		name = sanitizeExportFilename(name)
+		sum := sha256.Sum256([]byte(orig))
+		outPath := filepath.Join(base, hex.EncodeToString(sum[:6])+"-"+name)
+		if err := os.WriteFile(outPath, f.Content, 0o600); err != nil {
+			return nil, fmt.Errorf("import: restore file %q: %w", orig, err)
+		}
+		restored[orig] = outPath
+	}
+	return restored, nil
+}
+
+func applyRestoredFilePaths(reg *Registry, restored map[string]string) {
+	if reg == nil || len(restored) == 0 {
+		return
+	}
+	lookup := func(path string) string {
+		expanded, err := expandPath(path)
+		if err == nil {
+			if restoredPath := strings.TrimSpace(restored[expanded]); restoredPath != "" {
+				return restoredPath
+			}
+		}
+		if restoredPath := strings.TrimSpace(restored[strings.TrimSpace(path)]); restoredPath != "" {
+			return restoredPath
+		}
+		return path
+	}
+	for i := range reg.Clusters {
+		if strings.TrimSpace(reg.Clusters[i].KeyPath) != "" {
+			reg.Clusters[i].KeyPath = lookup(reg.Clusters[i].KeyPath)
+		}
+		if strings.TrimSpace(reg.Clusters[i].KeyPassphraseFile) != "" {
+			reg.Clusters[i].KeyPassphraseFile = lookup(reg.Clusters[i].KeyPassphraseFile)
+		}
+	}
+	for i := range reg.Backups.Targets {
+		if strings.TrimSpace(reg.Backups.Targets[i].SFTPKeyPath) != "" {
+			reg.Backups.Targets[i].SFTPKeyPath = lookup(reg.Backups.Targets[i].SFTPKeyPath)
+		}
+	}
+}
+
+func sanitizeExportFilename(name string) string {
+	var b strings.Builder
+	for _, r := range name {
+		switch {
+		case r >= 'a' && r <= 'z':
+			b.WriteRune(r)
+		case r >= 'A' && r <= 'Z':
+			b.WriteRune(r)
+		case r >= '0' && r <= '9':
+			b.WriteRune(r)
+		case r == '.', r == '_', r == '-':
+			b.WriteRune(r)
+		default:
+			b.WriteByte('_')
+		}
+	}
+	out := strings.Trim(b.String(), ".")
+	if out == "" {
+		return "secret"
+	}
+	return out
+}
+
+func mergeBackupConfig(current, incoming BackupConfig) BackupConfig {
+	if len(incoming.Targets) == 0 && len(incoming.Plans) == 0 {
+		return current
+	}
+	if current.Targets == nil {
+		current.Targets = []BackupTarget{}
+	}
+	if current.Plans == nil {
+		current.Plans = []BackupPlan{}
+	}
+
+	targetIDMap := map[string]string{}
+	targetByName := map[string]int{}
+	for i, t := range current.Targets {
+		if key := strings.ToLower(strings.TrimSpace(t.Name)); key != "" {
+			targetByName[key] = i
+		}
+	}
+	for _, inc := range incoming.Targets {
+		if strings.TrimSpace(inc.ID) == "" {
+			inc.ID = newClusterID()
+		}
+		key := strings.ToLower(strings.TrimSpace(inc.Name))
+		if key != "" {
+			if idx, ok := targetByName[key]; ok {
+				oldID := current.Targets[idx].ID
+				targetIDMap[inc.ID] = oldID
+				inc.ID = oldID
+				current.Targets[idx] = inc
+				continue
+			}
+		}
+		current.Targets = append(current.Targets, inc)
+		if key != "" {
+			targetByName[key] = len(current.Targets) - 1
+		}
+	}
+
+	planByName := map[string]int{}
+	for i, p := range current.Plans {
+		if key := strings.ToLower(strings.TrimSpace(p.Name)); key != "" {
+			planByName[key] = i
+		}
+	}
+	for _, inc := range incoming.Plans {
+		if mapped := strings.TrimSpace(targetIDMap[inc.TargetID]); mapped != "" {
+			inc.TargetID = mapped
+		}
+		if strings.TrimSpace(inc.ID) == "" {
+			inc.ID = newClusterID()
+		}
+		key := strings.ToLower(strings.TrimSpace(inc.Name))
+		if key != "" {
+			if idx, ok := planByName[key]; ok {
+				inc.ID = current.Plans[idx].ID
+				current.Plans[idx] = inc
+				continue
+			}
+		}
+		current.Plans = append(current.Plans, inc)
+		if key != "" {
+			planByName[key] = len(current.Plans) - 1
+		}
+	}
+	return current
+}
diff --git a/internal/cluster/model.go b/internal/cluster/model.go
new file mode 100644
index 0000000..a0723ac
--- /dev/null
+++ b/internal/cluster/model.go
@@ -0,0 +1,518 @@
+package cluster
+
+import (
+	"sort"
+	"strings"
+	"time"
+)
+
+const currentVersion = 8
+
+// AuthMethod describes how SSH authentication is performed.
+type AuthMethod string
+
+const (
+	AuthMethodPassword AuthMethod = "password"
+	AuthMethodKey      AuthMethod = "key"
+)
+
+// TransportMode selects how pxmon reaches the agent HTTP API on a node.
+type TransportMode string
+
+const (
+	// TransportDirect is the default: pxmon dials the agent's listen
+	// address over plain TCP from the local machine.
+	TransportDirect TransportMode = "direct"
+	// TransportIPFabric tunnels the agent HTTP call through the existing SSH
+	// connection. Meant for nodes on ipfabric-style networking where the node
+	// has no default outbound route and we must not touch its network config.
+	// The agent is expected to bind to 127.0.0.1 on the node.
+	TransportIPFabric TransportMode = "ipfabric"
+)
+
+func normalizeTransport(t TransportMode) TransportMode {
+	switch strings.ToLower(strings.TrimSpace(string(t))) {
+	case "ipfabric", "ip-fabric", "ip_fabric":
+		return TransportIPFabric
+	case "", "direct":
+		return TransportDirect
+	default:
+		return TransportDirect
+	}
+}
+
+// Registry is the local inventory of managed clusters (nodes).
+type Registry struct {
+	Version         int          `json:"version"`
+	ActiveClusterID string       `json:"active_cluster_id,omitempty"`
+	Telegram        Telegram     `json:"telegram,omitempty"`
+	Locker          Locker       `json:"locker,omitempty"`
+	Backups         BackupConfig `json:"backups,omitempty"`
+	Clusters        []Cluster    `json:"clusters"`
+}
+
+type BackupConfig struct {
+	Targets []BackupTarget `json:"targets,omitempty"`
+	Plans   []BackupPlan   `json:"plans,omitempty"`
+}
+
+type BackupTarget struct {
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	Type    string `json:"type"` // sftp|s3
+	Enabled bool   `json:"enabled"`
+
+	SFTPHost     string `json:"sftp_host,omitempty"`
+	SFTPPort     int    `json:"sftp_port,omitempty"`
+	SFTPUser     string `json:"sftp_user,omitempty"`
+	SFTPPassword string `json:"sftp_password,omitempty"`
+	SFTPKeyPath  string `json:"sftp_key_path,omitempty"`
+	SFTPBasePath string `json:"sftp_base_path,omitempty"`
+
+	S3Endpoint  string `json:"s3_endpoint,omitempty"`
+	S3Region    string `json:"s3_region,omitempty"`
+	S3Bucket    string `json:"s3_bucket,omitempty"`
+	S3Prefix    string `json:"s3_prefix,omitempty"`
+	S3AccessKey string `json:"s3_access_key,omitempty"`
+	S3SecretKey string `json:"s3_secret_key,omitempty"`
+	S3UseSSL    bool   `json:"s3_use_ssl,omitempty"`
+	S3PathStyle bool   `json:"s3_path_style,omitempty"`
+
+	CreatedAt time.Time `json:"created_at"`
+	UpdatedAt time.Time `json:"updated_at"`
+}
+
+type BackupPlan struct {
+	ID          string    `json:"id"`
+	Name        string    `json:"name"`
+	Cluster     string    `json:"cluster,omitempty"`
+	TargetID    string    `json:"target_id"`
+	Paths       []string  `json:"paths"`
+	Every       string    `json:"every,omitempty"`
+	Enabled     bool      `json:"enabled"`
+	RetainDays  int       `json:"retain_days,omitempty"`
+	Compress    bool      `json:"compress"`
+	LastRunAt   time.Time `json:"last_run_at,omitempty"`
+	LastStatus  string    `json:"last_status,omitempty"`
+	LastArchive string    `json:"last_archive,omitempty"`
+	LastError   string    `json:"last_error,omitempty"`
+	CreatedAt   time.Time `json:"created_at"`
+	UpdatedAt   time.Time `json:"updated_at"`
+}
+
+// Telegram stores Telegram bot integration settings.
+type Telegram struct {
+	Enabled        bool      `json:"enabled,omitempty"`
+	Token          string    `json:"token,omitempty"`
+	AllowedUserIDs []int64   `json:"allowed_user_ids,omitempty"`
+	UpdatedAt      time.Time `json:"updated_at,omitempty"`
+}
+
+// Locker stores global UI/CLI lock settings.
+type Locker struct {
+	Enabled      bool      `json:"enabled,omitempty"`
+	PasswordHash string    `json:"password_hash,omitempty"`
+	UpdatedAt    time.Time `json:"updated_at,omitempty"`
+}
+
+// Cluster describes one managed node that we can reach over SSH.
+type Cluster struct {
+	ID                string              `json:"id"`
+	Name              string              `json:"name"`
+	Host              string              `json:"host"`
+	Port              int                 `json:"port"`
+	User              string              `json:"user"`
+	Transport         TransportMode       `json:"transport,omitempty"`
+	AuthMethod        AuthMethod          `json:"auth_method"`
+	Password          string              `json:"password,omitempty"`
+	KeyPath           string              `json:"key_path,omitempty"`
+	KeyPassphrase     string              `json:"key_passphrase,omitempty"`
+	KeyPassphraseFile string              `json:"key_passphrase_file,omitempty"`
+	InsecureHostKey   bool                `json:"insecure_host_key,omitempty"`
+	Alerts            AlertPolicy         `json:"alerts"`
+	VMAlerts          VMAlertPolicy       `json:"vm_alerts,omitempty"`
+	AlertRouting      AlertRoutingPolicy  `json:"alert_routing,omitempty"`
+	RepoTunnel        RepoTunnelState     `json:"repo_tunnel,omitempty"`
+	RunbookTrigger    RunbookTrigger      `json:"runbook_trigger,omitempty"`
+	Drift             DriftControl        `json:"drift,omitempty"`
+	Tags              []string            `json:"tags,omitempty"`
+	KVMTags           map[string][]string `json:"kvm_tags,omitempty"`
+	Agent             AgentInstall        `json:"agent,omitempty"`
+	Software          SoftwareInfo        `json:"software,omitempty"`
+	CreatedAt         time.Time           `json:"created_at"`
+	UpdatedAt         time.Time           `json:"updated_at"`
+}
+
+// AlertPolicy defines warning thresholds used by CLI monitor mode.
+type AlertPolicy struct {
+	CPUWarnPercent         float64  `json:"cpu_warn_percent"`
+	RAMWarnPercent         float64  `json:"ram_warn_percent"`
+	SwapWarnPercent        float64  `json:"swap_warn_percent"`
+	DiskWarnPercent        float64  `json:"disk_warn_percent"`
+	NetWarnMbps            float64  `json:"net_warn_mbps"`
+	NetSustainEnabled      bool     `json:"net_sustain_enabled,omitempty"`
+	NetSustainIface        string   `json:"net_sustain_iface,omitempty"`
+	NetSustainInclude      []string `json:"net_sustain_include,omitempty"`
+	NetSustainExclude      []string `json:"net_sustain_exclude,omitempty"`
+	NetSustainMbps         float64  `json:"net_sustain_mbps,omitempty"`
+	NetSustainMinutes      int      `json:"net_sustain_minutes,omitempty"`
+	NetSustainCooldownMins int      `json:"net_sustain_cooldown_mins,omitempty"`
+}
+
+// VMAlertPolicy controls KVM VM-state alerting per cluster.
+type VMAlertPolicy struct {
+	Enabled       bool `json:"enabled,omitempty"`
+	WarnOnShutoff bool `json:"warn_on_shutoff,omitempty"`
+	MinRunning    int  `json:"min_running,omitempty"`
+}
+
+// AlertRoutingPolicy controls delivery behavior for alerts (e.g. Telegram).
+type AlertRoutingPolicy struct {
+	CriticalImmediate bool `json:"critical_immediate,omitempty"`
+	WarningBatchMins  int  `json:"warning_batch_mins,omitempty"`
+}
+
+// RunbookTrigger controls automatic runbook execution on selected events.
+type RunbookTrigger struct {
+	Enabled       bool      `json:"enabled,omitempty"`
+	OnVMShutoff   bool      `json:"on_vm_shutoff,omitempty"`
+	RunbookID     string    `json:"runbook_id,omitempty"`
+	CooldownMins  int       `json:"cooldown_mins,omitempty"`
+	LastTriggered time.Time `json:"last_triggered,omitempty"`
+}
+
+// DriftBaseline stores expected values for drift comparisons.
+type DriftBaseline struct {
+	Enabled      bool      `json:"enabled,omitempty"`
+	SetAt        time.Time `json:"set_at,omitempty"`
+	AgentVersion string    `json:"agent_version,omitempty"`
+	Software     string    `json:"software,omitempty"`
+}
+
+// DriftControl stores baseline and per-issue acknowledgement windows.
+type DriftControl struct {
+	Baseline DriftBaseline        `json:"baseline,omitempty"`
+	AckUntil map[string]time.Time `json:"ack_until,omitempty"`
+}
+
+// AgentInstall describes remote pxmon-agent installation details.
+type AgentInstall struct {
+	Installed       bool      `json:"installed"`
+	Version         string    `json:"version,omitempty"`
+	RemoteBinary    string    `json:"remote_binary,omitempty"`
+	RemoteConfig    string    `json:"remote_config,omitempty"`
+	RemoteLog       string    `json:"remote_log,omitempty"`
+	RemotePIDFile   string    `json:"remote_pid_file,omitempty"`
+	ListenAddress   string    `json:"listen_address,omitempty"`
+	Port            int       `json:"port,omitempty"`
+	Token           string    `json:"token,omitempty"`
+	RequestSecret   string    `json:"request_secret,omitempty"`
+	TLSEnabled      bool      `json:"tls_enabled,omitempty"`
+	TLSCertPath     string    `json:"tls_cert_path,omitempty"`
+	TLSKeyPath      string    `json:"tls_key_path,omitempty"`
+	TLSFingerprint  string    `json:"tls_fingerprint,omitempty"`
+	LastBootstrapAt time.Time `json:"last_bootstrap_at,omitempty"`
+}
+
+// SoftwareInfo describes discovered software/plugins on a node.
+type SoftwareInfo struct {
+	DetectedAt time.Time         `json:"detected_at,omitempty"`
+	Bird       bool              `json:"bird,omitempty"`
+	FRR        bool              `json:"frr,omitempty"`
+	KVM        bool              `json:"kvm,omitempty"`
+	LXC        bool              `json:"lxc,omitempty"`
+	LXD        bool              `json:"lxd,omitempty"`
+	Versions   map[string]string `json:"versions,omitempty"`
+}
+
+func (s SoftwareInfo) SupportedList() []string {
+	out := make([]string, 0, 5)
+	if s.Bird {
+		out = append(out, "bird")
+	}
+	if s.FRR {
+		out = append(out, "frr")
+	}
+	if s.KVM {
+		out = append(out, "kvm")
+	}
+	if s.LXC {
+		out = append(out, "lxc")
+	}
+	if s.LXD {
+		out = append(out, "lxd")
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (s SoftwareInfo) Summary() string {
+	list := s.SupportedList()
+	if len(list) == 0 {
+		if !s.DetectedAt.IsZero() {
+			return "none"
+		}
+		return "-"
+	}
+	return strings.Join(list, ",")
+}
+
+func newRegistry() Registry {
+	return Registry{
+		Version:  currentVersion,
+		Backups:  normalizeBackupConfig(BackupConfig{}),
+		Clusters: []Cluster{},
+	}
+}
+
+func defaultAlertPolicy() AlertPolicy {
+	return AlertPolicy{
+		CPUWarnPercent:  85,
+		RAMWarnPercent:  90,
+		SwapWarnPercent: 80,
+		DiskWarnPercent: 90,
+		NetWarnMbps:     300,
+	}
+}
+
+func defaultVMAlertPolicy() VMAlertPolicy {
+	return VMAlertPolicy{
+		Enabled:       false,
+		WarnOnShutoff: true,
+		MinRunning:    1,
+	}
+}
+
+func defaultAlertRoutingPolicy() AlertRoutingPolicy {
+	return AlertRoutingPolicy{
+		CriticalImmediate: true,
+		WarningBatchMins:  5,
+	}
+}
+
+func defaultRunbookTrigger() RunbookTrigger {
+	return RunbookTrigger{
+		Enabled:      false,
+		OnVMShutoff:  true,
+		CooldownMins: 30,
+	}
+}
+
+func ensureAlertPolicy(p AlertPolicy) AlertPolicy {
+	d := defaultAlertPolicy()
+	if p.CPUWarnPercent <= 0 {
+		p.CPUWarnPercent = d.CPUWarnPercent
+	}
+	if p.RAMWarnPercent <= 0 {
+		p.RAMWarnPercent = d.RAMWarnPercent
+	}
+	if p.SwapWarnPercent <= 0 {
+		p.SwapWarnPercent = d.SwapWarnPercent
+	}
+	if p.DiskWarnPercent <= 0 {
+		p.DiskWarnPercent = d.DiskWarnPercent
+	}
+	if p.NetWarnMbps <= 0 {
+		p.NetWarnMbps = d.NetWarnMbps
+	}
+	if p.NetSustainEnabled {
+		if p.NetSustainMbps <= 0 {
+			p.NetSustainMbps = d.NetWarnMbps
+		}
+		if p.NetSustainMinutes <= 0 {
+			p.NetSustainMinutes = 60
+		}
+		if p.NetSustainCooldownMins <= 0 {
+			p.NetSustainCooldownMins = 30
+		}
+	}
+	return p
+}
+
+func ensureVMAlertPolicy(p VMAlertPolicy) VMAlertPolicy {
+	d := defaultVMAlertPolicy()
+	wasZero := p == (VMAlertPolicy{})
+	if p.MinRunning <= 0 {
+		p.MinRunning = d.MinRunning
+	}
+	if !p.WarnOnShutoff {
+		// Keep explicit false if user set it, but default to true for zero-value
+		// policy loaded from old configs.
+		if wasZero {
+			p.WarnOnShutoff = d.WarnOnShutoff
+		}
+	}
+	return p
+}
+
+func ensureAlertRoutingPolicy(p AlertRoutingPolicy) AlertRoutingPolicy {
+	d := defaultAlertRoutingPolicy()
+	if p.WarningBatchMins <= 0 {
+		p.WarningBatchMins = d.WarningBatchMins
+	}
+	// default true when unset
+	if !p.CriticalImmediate && p == (AlertRoutingPolicy{}) {
+		p.CriticalImmediate = d.CriticalImmediate
+	}
+	return p
+}
+
+func ensureRunbookTrigger(p RunbookTrigger) RunbookTrigger {
+	d := defaultRunbookTrigger()
+	if p.CooldownMins <= 0 {
+		p.CooldownMins = d.CooldownMins
+	}
+	if !p.OnVMShutoff && p == (RunbookTrigger{}) {
+		p.OnVMShutoff = d.OnVMShutoff
+	}
+	p.RunbookID = strings.TrimSpace(p.RunbookID)
+	return p
+}
+
+func normalizeBackupConfig(cfg BackupConfig) BackupConfig {
+	if cfg.Targets == nil {
+		cfg.Targets = []BackupTarget{}
+	}
+	if cfg.Plans == nil {
+		cfg.Plans = []BackupPlan{}
+	}
+	for i := range cfg.Targets {
+		t := &cfg.Targets[i]
+		t.ID = strings.TrimSpace(t.ID)
+		t.Name = strings.TrimSpace(t.Name)
+		t.Type = strings.ToLower(strings.TrimSpace(t.Type))
+		if t.SFTPPort <= 0 {
+			t.SFTPPort = 22
+		}
+		if t.ID == "" {
+			t.ID = newClusterID()
+		}
+		if t.Name == "" {
+			t.Name = t.ID
+		}
+		if t.Type != "sftp" && t.Type != "s3" {
+			t.Type = "sftp"
+		}
+		if !t.Enabled && t.CreatedAt.IsZero() {
+			t.Enabled = true
+		}
+		if t.CreatedAt.IsZero() {
+			t.CreatedAt = time.Now().UTC()
+		}
+		if t.UpdatedAt.IsZero() {
+			t.UpdatedAt = t.CreatedAt
+		}
+	}
+	for i := range cfg.Plans {
+		p := &cfg.Plans[i]
+		p.ID = strings.TrimSpace(p.ID)
+		p.Name = strings.TrimSpace(p.Name)
+		p.Cluster = strings.TrimSpace(p.Cluster)
+		p.TargetID = strings.TrimSpace(p.TargetID)
+		if p.ID == "" {
+			p.ID = newClusterID()
+		}
+		if p.Name == "" {
+			p.Name = p.ID
+		}
+		if p.Paths == nil {
+			p.Paths = []string{}
+		}
+		if p.Every == "" {
+			p.Every = "24h"
+		}
+		if p.RetainDays <= 0 {
+			p.RetainDays = 30
+		}
+		if !p.Compress {
+			p.Compress = true
+		}
+		if !p.Enabled && p.CreatedAt.IsZero() {
+			p.Enabled = true
+		}
+		if p.CreatedAt.IsZero() {
+			p.CreatedAt = time.Now().UTC()
+		}
+		if p.UpdatedAt.IsZero() {
+			p.UpdatedAt = p.CreatedAt
+		}
+	}
+	return cfg
+}
+
+func normalizeDriftControl(d DriftControl) DriftControl {
+	if len(d.AckUntil) == 0 {
+		d.AckUntil = nil
+		return d
+	}
+	out := make(map[string]time.Time, len(d.AckUntil))
+	for k, v := range d.AckUntil {
+		n := strings.ToLower(strings.TrimSpace(k))
+		if n == "" || v.IsZero() {
+			continue
+		}
+		out[n] = v.UTC()
+	}
+	if len(out) == 0 {
+		d.AckUntil = nil
+	} else {
+		d.AckUntil = out
+	}
+	d.Baseline.AgentVersion = strings.TrimSpace(d.Baseline.AgentVersion)
+	d.Baseline.Software = strings.TrimSpace(d.Baseline.Software)
+	return d
+}
+
+func normalizeTagList(tags []string) []string {
+	if len(tags) == 0 {
+		return nil
+	}
+	seen := make(map[string]struct{}, len(tags))
+	out := make([]string, 0, len(tags))
+	for _, t := range tags {
+		n := strings.ToLower(strings.TrimSpace(t))
+		if n == "" {
+			continue
+		}
+		if _, ok := seen[n]; ok {
+			continue
+		}
+		seen[n] = struct{}{}
+		out = append(out, n)
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	sort.Strings(out)
+	return out
+}
+
+func normalizeVMTagMap(m map[string][]string) map[string][]string {
+	if len(m) == 0 {
+		return nil
+	}
+	out := make(map[string][]string, len(m))
+	for vm, tags := range m {
+		vmName := strings.TrimSpace(vm)
+		if vmName == "" {
+			continue
+		}
+		norm := normalizeTagList(tags)
+		if len(norm) == 0 {
+			continue
+		}
+		out[vmName] = norm
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+func normalizeLocker(cfg Locker) Locker {
+	cfg.PasswordHash = strings.TrimSpace(cfg.PasswordHash)
+	if cfg.PasswordHash == "" {
+		cfg.Enabled = false
+	}
+	return cfg
+}
diff --git a/internal/cluster/p95.go b/internal/cluster/p95.go
new file mode 100644
index 0000000..bc356f3
--- /dev/null
+++ b/internal/cluster/p95.go
@@ -0,0 +1,82 @@
+package cluster
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"strings"
+
+	"pxmon/internal/history"
+)
+
+type InterfaceP95Snapshot struct {
+	ClusterName string                    `json:"cluster"`
+	ClusterID   string                    `json:"cluster_id"`
+	Interface   string                    `json:"interface"`
+	Range       history.RangeShortcut     `json:"range"`
+	Samples     int                       `json:"samples"`
+	P95Mbps     float64                   `json:"p95_mbps"`
+	AvgMbps     float64                   `json:"avg_mbps"`
+	MaxMbps     float64                   `json:"max_mbps"`
+	Series      []history.NodeSamplePoint `json:"series,omitempty"`
+}
+
+func (s *Service) CollectInterfaceP95(selector, iface string, rng history.RangeShortcut) (InterfaceP95Snapshot, error) {
+	iface = strings.TrimSpace(iface)
+	if iface == "" {
+		return InterfaceP95Snapshot{}, errors.New("interface is required")
+	}
+	c, err := s.Get(selector)
+	if err != nil {
+		return InterfaceP95Snapshot{}, err
+	}
+	store := s.NetworkStore()
+	if store == nil {
+		return InterfaceP95Snapshot{}, errors.New("history store not configured")
+	}
+	snaps, err := store.Load(c.ID, rng.Since(s.now()))
+	if err != nil {
+		return InterfaceP95Snapshot{}, err
+	}
+	series := history.AggregateNodeSeries(snaps, iface)
+	out := InterfaceP95Snapshot{
+		ClusterName: c.Name,
+		ClusterID:   c.ID,
+		Interface:   iface,
+		Range:       rng,
+		Samples:     len(series),
+		P95Mbps:     history.PercentileMbps(series, 95),
+		Series:      series,
+	}
+	if len(series) == 0 {
+		return out, nil
+	}
+	var sum, maxV float64
+	for _, p := range series {
+		sum += p.TotalMbps
+		if p.TotalMbps > maxV {
+			maxV = p.TotalMbps
+		}
+	}
+	out.AvgMbps = sum / float64(len(series))
+	out.MaxMbps = maxV
+	return out, nil
+}
+
+func (s *Service) RenderInterfaceP95GraphPNG(snap InterfaceP95Snapshot) ([]byte, error) {
+	if len(snap.Series) == 0 {
+		return history.RenderNodeNetworkPNG([]history.NodeSamplePoint{}, history.ChartOptions{
+			Title:    fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
+			Subtitle: "No samples",
+		})
+	}
+	subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps", snap.P95Mbps, snap.MaxMbps, snap.AvgMbps)
+	if math.IsNaN(snap.P95Mbps) {
+		subtitle = "No samples"
+	}
+	return history.RenderNodeNetworkPNG(snap.Series, history.ChartOptions{
+		Title:      fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
+		Subtitle:   subtitle,
+		Percentile: 95,
+	})
+}
diff --git a/internal/cluster/policies.go b/internal/cluster/policies.go
new file mode 100644
index 0000000..5cf01dd
--- /dev/null
+++ b/internal/cluster/policies.go
@@ -0,0 +1,152 @@
+package cluster
+
+import (
+	"errors"
+	"strings"
+	"time"
+)
+
+func (s *Service) GetAlertRouting(selector string) (AlertRoutingPolicy, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return AlertRoutingPolicy{}, err
+	}
+	return ensureAlertRoutingPolicy(c.AlertRouting), nil
+}
+
+func (s *Service) SetAlertRouting(selector string, p AlertRoutingPolicy) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	c.AlertRouting = ensureAlertRoutingPolicy(p)
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("alert.routing", c.Name, "updated")
+	return c, nil
+}
+
+func (s *Service) GetRunbookTrigger(selector string) (RunbookTrigger, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return RunbookTrigger{}, err
+	}
+	return ensureRunbookTrigger(c.RunbookTrigger), nil
+}
+
+func (s *Service) SetRunbookTrigger(selector string, p RunbookTrigger) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	c.RunbookTrigger = ensureRunbookTrigger(p)
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("runbook.trigger", c.Name, "updated")
+	return c, nil
+}
+
+func (s *Service) TouchRunbookTrigger(selector string, when time.Time) error {
+	reg, err := s.store.Load()
+	if err != nil {
+		return err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return err
+	}
+	tr := ensureRunbookTrigger(c.RunbookTrigger)
+	tr.LastTriggered = when.UTC()
+	c.RunbookTrigger = tr
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	return s.store.Save(reg)
+}
+
+func (s *Service) SetDriftBaseline(selector string) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	d := normalizeDriftControl(c.Drift)
+	d.Baseline = DriftBaseline{
+		Enabled:      true,
+		SetAt:        s.now().UTC(),
+		AgentVersion: strings.TrimSpace(c.Agent.Version),
+		Software:     strings.TrimSpace(c.Software.Summary()),
+	}
+	c.Drift = d
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("drift.baseline", c.Name, "set")
+	return c, nil
+}
+
+func (s *Service) GetDriftControl(selector string) (DriftControl, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return DriftControl{}, err
+	}
+	return normalizeDriftControl(c.Drift), nil
+}
+
+func (s *Service) AckDriftIssue(selector, issueKind string, until time.Time) (Cluster, error) {
+	kind := strings.ToLower(strings.TrimSpace(issueKind))
+	if kind == "" {
+		return Cluster{}, errors.New("issue kind is required")
+	}
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	d := normalizeDriftControl(c.Drift)
+	if d.AckUntil == nil {
+		d.AckUntil = map[string]time.Time{}
+	}
+	d.AckUntil[kind] = until.UTC()
+	c.Drift = d
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("drift.ack", c.Name, kind+" until="+until.UTC().Format(time.RFC3339))
+	return c, nil
+}
+
+func (s *Service) IsDriftIssueAcked(c Cluster, kind string, now time.Time) bool {
+	d := normalizeDriftControl(c.Drift)
+	if len(d.AckUntil) == 0 {
+		return false
+	}
+	u, ok := d.AckUntil[strings.ToLower(strings.TrimSpace(kind))]
+	if !ok {
+		return false
+	}
+	return now.UTC().Before(u)
+}
diff --git a/internal/cluster/repo_tunnel.go b/internal/cluster/repo_tunnel.go
new file mode 100644
index 0000000..fa89452
--- /dev/null
+++ b/internal/cluster/repo_tunnel.go
@@ -0,0 +1,413 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+type RepoTunnelOptions struct {
+	Gateway        string
+	GatewayIP      string
+	Table          int
+	Priority       int
+	PackageManager string
+	Command        string
+	KeepEnabled    bool
+	NoRule         bool
+}
+
+type RepoTunnelState struct {
+	Enabled bool   `json:"enabled"`
+	Proxy   string `json:"proxy,omitempty"`
+	Source  string `json:"source,omitempty"`
+}
+
+func (s *Service) RepoTunnelEnable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
+	script, err := repoTunnelEnableScript(opts)
+	if err != nil {
+		return "", err
+	}
+	out, err := s.RunRemoteShell(ctx, selector, script)
+	if err != nil {
+		return "", err
+	}
+	gw, err := parseRepoTunnelGateway(opts.Gateway)
+	if err == nil {
+		_ = s.updateRepoTunnelState(selector, RepoTunnelState{
+			Enabled: true,
+			Proxy:   gw.proxyURL,
+			Source:  strings.ToLower(strings.TrimSpace(opts.PackageManager)),
+		})
+	}
+	return out, nil
+}
+
+func (s *Service) RepoTunnelDisable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
+	script, err := repoTunnelDisableScript(opts)
+	if err != nil {
+		return "", err
+	}
+	out, err := s.RunRemoteShell(ctx, selector, script)
+	if err != nil {
+		return "", err
+	}
+	_ = s.updateRepoTunnelState(selector, RepoTunnelState{})
+	return out, nil
+}
+
+func (s *Service) RepoTunnelState(ctx context.Context, selector string) (RepoTunnelState, error) {
+	out, err := s.RunRemoteShell(ctx, selector, repoTunnelDetectScript())
+	if err != nil {
+		return RepoTunnelState{}, err
+	}
+	state := RepoTunnelState{}
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		switch {
+		case strings.HasPrefix(line, "proxy="):
+			state.Proxy = strings.TrimSpace(strings.TrimPrefix(line, "proxy="))
+		case strings.HasPrefix(line, "source="):
+			state.Source = strings.TrimSpace(strings.TrimPrefix(line, "source="))
+		}
+	}
+	state.Enabled = state.Proxy != ""
+	return state, nil
+}
+
+func (s *Service) updateRepoTunnelState(selector string, state RepoTunnelState) error {
+	reg, err := s.store.Load()
+	if err != nil {
+		return err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return err
+	}
+	c.RepoTunnel = state
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	return s.store.Save(reg)
+}
+
+func (s *Service) RepoTunnelStatus(ctx context.Context, selector string) (string, error) {
+	return s.RunRemoteShell(ctx, selector, repoTunnelStatusScript())
+}
+
+func (s *Service) RepoTunnelInstall(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
+	if strings.TrimSpace(opts.Command) == "" {
+		return "", errors.New("install command is required")
+	}
+	enableScript, err := repoTunnelEnableScript(opts)
+	if err != nil {
+		return "", err
+	}
+	disableScript, err := repoTunnelDisableScript(opts)
+	if err != nil {
+		return "", err
+	}
+	body := enableScript + "\n" + strings.TrimSpace(opts.Command) + "\n"
+	if !opts.KeepEnabled {
+		body = enableScript + "\ncleanup_pxmon_repo_tunnel() {\n" + disableScript + "\n}\ntrap cleanup_pxmon_repo_tunnel EXIT\n" + strings.TrimSpace(opts.Command) + "\n"
+	}
+	return s.RunRemoteShell(ctx, selector, body)
+}
+
+func RepoTunnelGatewayScript(port int, allowCIDRs []string) (string, error) {
+	if port == 0 {
+		port = 3128
+	}
+	if port < 1 || port > 65535 {
+		return "", errors.New("--port must be in range 1..65535")
+	}
+	if len(allowCIDRs) == 0 {
+		return "", errors.New("at least one --allow CIDR/IP is required")
+	}
+	aclParts := make([]string, 0, len(allowCIDRs))
+	for _, raw := range allowCIDRs {
+		v := strings.TrimSpace(raw)
+		if v == "" {
+			continue
+		}
+		if !validSquidSrcACL(v) {
+			return "", fmt.Errorf("invalid --allow %q; use an IP or CIDR without spaces", raw)
+		}
+		aclParts = append(aclParts, v)
+	}
+	if len(aclParts) == 0 {
+		return "", errors.New("at least one --allow CIDR/IP is required")
+	}
+
+	return fmt.Sprintf(`set -eu
+if command -v dnf >/dev/null 2>&1; then
+  dnf install -y squid
+elif command -v yum >/dev/null 2>&1; then
+  yum install -y squid
+elif command -v apt-get >/dev/null 2>&1; then
+  apt-get update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y squid
+else
+  echo "no supported package manager found for squid install" >&2
+  exit 1
+fi
+
+conf=/etc/squid/squid.conf
+cp -a "$conf" "$conf.pxmon-bak.$(date +%%Y%%m%%d%%H%%M%%S)"
+awk '
+  /# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
+  /# END PXMON REPO TUNNEL/ {skip=0; next}
+  !skip {print}
+' "$conf" > "$conf.tmp"
+mv "$conf.tmp" "$conf"
+
+block=$(mktemp)
+{
+  echo "# BEGIN PXMON REPO TUNNEL"
+  if ! grep -Eq "^http_port[[:space:]]+([^[:space:]]+:)?%d\b" "$conf"; then
+    echo "http_port %d"
+  fi
+  echo "acl pxmon_repo_tunnel src %s"
+  echo "http_access allow pxmon_repo_tunnel"
+  echo "# END PXMON REPO TUNNEL"
+} > "$block"
+
+if grep -q "^http_access deny all" "$conf"; then
+  awk -v block="$block" '
+    BEGIN {while ((getline line < block) > 0) b = b line "\n"; close(block); inserted=0}
+    /^http_access deny all/ && !inserted {printf "%%s", b; inserted=1}
+    {print}
+    END {if (!inserted) printf "%%s", b}
+  ' "$conf" > "$conf.tmp"
+  mv "$conf.tmp" "$conf"
+else
+  cat "$block" >> "$conf"
+fi
+rm -f "$block"
+
+systemctl enable --now squid
+systemctl restart squid
+if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
+  firewall-cmd --add-port=%d/tcp --permanent
+  firewall-cmd --reload
+fi
+echo "pxmon repo gateway ready on port %d"
+`, port, port, strings.Join(aclParts, " "), port, port), nil
+}
+
+func validSquidSrcACL(v string) bool {
+	for _, r := range v {
+		if r >= 'a' && r <= 'z' {
+			continue
+		}
+		if r >= 'A' && r <= 'Z' {
+			continue
+		}
+		if r >= '0' && r <= '9' {
+			continue
+		}
+		switch r {
+		case '.', ':', '/', '_', '-':
+			continue
+		default:
+			return false
+		}
+	}
+	return v != ""
+}
+
+func repoTunnelEnableScript(opts RepoTunnelOptions) (string, error) {
+	gw, err := parseRepoTunnelGateway(opts.Gateway)
+	if err != nil {
+		return "", err
+	}
+	if !opts.NoRule && opts.Table <= 0 {
+		return "", errors.New("--table is required unless --no-rule is used")
+	}
+	manager := strings.ToLower(strings.TrimSpace(opts.PackageManager))
+	if manager == "" {
+		manager = "auto"
+	}
+	if manager != "auto" && manager != "apt" && manager != "dnf" && manager != "yum" {
+		return "", fmt.Errorf("unsupported --manager %q", opts.PackageManager)
+	}
+	ruleLine := ""
+	if !opts.NoRule {
+		addCmd := fmt.Sprintf("ip rule add to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Table)
+		if opts.Priority > 0 {
+			addCmd = fmt.Sprintf("ip rule add priority %d to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Priority, opts.Table)
+		}
+		ruleLine = fmt.Sprintf(`
+if ! ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; then
+  %s
+fi`, opts.Table, addCmd)
+	}
+	return fmt.Sprintf(`set -eu
+PXMON_GATEWAY_HOST=%s
+PXMON_GATEWAY_IP=%s
+PXMON_PROXY_URL=%s
+PXMON_MANAGER=%s
+if [ -z "$PXMON_GATEWAY_IP" ]; then
+  PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
+fi
+if [ -z "$PXMON_GATEWAY_IP" ]; then
+  echo "cannot resolve repo gateway: $PXMON_GATEWAY_HOST" >&2
+  exit 1
+fi
+%s
+pxmon_repo_manager="$PXMON_MANAGER"
+if [ "$pxmon_repo_manager" = "auto" ]; then
+  if command -v apt-get >/dev/null 2>&1; then pxmon_repo_manager=apt
+  elif command -v dnf >/dev/null 2>&1; then pxmon_repo_manager=dnf
+  elif command -v yum >/dev/null 2>&1; then pxmon_repo_manager=yum
+  else echo "no supported package manager found" >&2; exit 1
+  fi
+fi
+case "$pxmon_repo_manager" in
+  apt)
+    mkdir -p /etc/apt/apt.conf.d
+    cat > /etc/apt/apt.conf.d/99-pxmon-repo-tunnel < "$conf.tmp"
+    mv "$conf.tmp" "$conf"
+    {
+      echo "# BEGIN PXMON REPO TUNNEL"
+      echo "proxy=$PXMON_PROXY_URL"
+      echo "# END PXMON REPO TUNNEL"
+    } >> "$conf"
+    ;;
+esac
+echo "pxmon repo tunnel enabled: proxy=$PXMON_PROXY_URL gateway_ip=$PXMON_GATEWAY_IP manager=$pxmon_repo_manager"
+`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), shellQuote(gw.proxyURL), shellQuote(manager), ruleLine), nil
+}
+
+func repoTunnelDisableScript(opts RepoTunnelOptions) (string, error) {
+	gw, err := parseRepoTunnelGateway(opts.Gateway)
+	if err != nil && !opts.NoRule {
+		return "", err
+	}
+	ruleLine := ""
+	if !opts.NoRule {
+		if opts.Table <= 0 {
+			return "", errors.New("--table is required unless --no-rule is used")
+		}
+		ruleLine = fmt.Sprintf(`
+PXMON_GATEWAY_HOST=%s
+PXMON_GATEWAY_IP=%s
+if [ -z "$PXMON_GATEWAY_IP" ]; then
+  PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
+fi
+if [ -n "$PXMON_GATEWAY_IP" ]; then
+  while ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; do
+    ip rule del to "$PXMON_GATEWAY_IP/32" table %d 2>/dev/null || break
+  done
+fi`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), opts.Table, opts.Table)
+	}
+	return fmt.Sprintf(`set -eu
+rm -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel
+for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
+  if [ -f "$conf" ]; then
+    awk '
+      /# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
+      /# END PXMON REPO TUNNEL/ {skip=0; next}
+      !skip {print}
+    ' "$conf" > "$conf.tmp"
+    mv "$conf.tmp" "$conf"
+  fi
+done
+%s
+echo "pxmon repo tunnel disabled"
+`, ruleLine), nil
+}
+
+func repoTunnelStatusScript() string {
+	return `set -eu
+echo "== ip rules =="
+ip rule show | grep -E "lookup|table" || true
+echo
+echo "== apt proxy =="
+[ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ] && cat /etc/apt/apt.conf.d/99-pxmon-repo-tunnel || echo "(none)"
+echo
+echo "== dnf/yum proxy =="
+for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
+  [ -f "$conf" ] || continue
+  echo "-- $conf"
+  awk '/# BEGIN PXMON REPO TUNNEL/,/# END PXMON REPO TUNNEL/ {print}' "$conf"
+done`
+}
+
+func repoTunnelDetectScript() string {
+	return `set -eu
+if [ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ]; then
+  proxy=$(sed -n 's/.*Proxy[[:space:]]*"\([^"]*\)".*/\1/p' /etc/apt/apt.conf.d/99-pxmon-repo-tunnel | head -1)
+  [ -n "$proxy" ] && printf 'proxy=%s\nsource=apt\n' "$proxy" && exit 0
+fi
+for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
+  [ -f "$conf" ] || continue
+  proxy=$(awk '
+    /# BEGIN PXMON REPO TUNNEL/ {inside=1; next}
+    /# END PXMON REPO TUNNEL/ {inside=0; next}
+    inside && /^proxy[[:space:]]*=/ {
+      sub(/^[^=]*=/, "")
+      gsub(/^[[:space:]]+|[[:space:]]+$/, "")
+      print
+      exit
+    }
+  ' "$conf")
+  [ -n "$proxy" ] && printf 'proxy=%s\nsource=%s\n' "$proxy" "$conf" && exit 0
+done
+exit 0`
+}
+
+type repoTunnelGateway struct {
+	host     string
+	port     int
+	proxyURL string
+}
+
+func parseRepoTunnelGateway(raw string) (repoTunnelGateway, error) {
+	v := strings.TrimSpace(raw)
+	if v == "" {
+		return repoTunnelGateway{}, errors.New("--gateway is required")
+	}
+	if strings.HasPrefix(v, "http://") {
+		v = strings.TrimPrefix(v, "http://")
+	}
+	if strings.HasPrefix(v, "https://") {
+		return repoTunnelGateway{}, errors.New("--gateway must be an http proxy endpoint, not https")
+	}
+	host, portRaw, err := net.SplitHostPort(v)
+	if err != nil {
+		if strings.Count(v, ":") > 1 {
+			return repoTunnelGateway{}, fmt.Errorf("invalid --gateway %q; use host:port or [ipv6]:port", raw)
+		}
+		host = v
+		portRaw = "3128"
+	}
+	host = strings.Trim(host, "[]")
+	if strings.TrimSpace(host) == "" {
+		return repoTunnelGateway{}, errors.New("--gateway host is empty")
+	}
+	port, err := strconv.Atoi(portRaw)
+	if err != nil || port < 1 || port > 65535 {
+		return repoTunnelGateway{}, fmt.Errorf("invalid --gateway port %q", portRaw)
+	}
+	return repoTunnelGateway{
+		host:     host,
+		port:     port,
+		proxyURL: "http://" + net.JoinHostPort(host, strconv.Itoa(port)),
+	}, nil
+}
diff --git a/internal/cluster/runbook.go b/internal/cluster/runbook.go
new file mode 100644
index 0000000..14e08e3
--- /dev/null
+++ b/internal/cluster/runbook.go
@@ -0,0 +1,187 @@
+package cluster
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+type RunbookStep struct {
+	Title   string `json:"title"`
+	Command string `json:"command,omitempty"`
+	Note    string `json:"note,omitempty"`
+}
+
+type Runbook struct {
+	ID          string        `json:"id"`
+	Name        string        `json:"name"`
+	Description string        `json:"description,omitempty"`
+	Steps       []RunbookStep `json:"steps"`
+	BuiltIn     bool          `json:"built_in,omitempty"`
+	CreatedAt   time.Time     `json:"created_at,omitempty"`
+	UpdatedAt   time.Time     `json:"updated_at,omitempty"`
+}
+
+func builtinRunbooks() []Runbook {
+	return []Runbook{
+		{
+			ID:          "vm-health-check",
+			Name:        "VM Health Check",
+			Description: "Quick validation of agent, VM inventory and VM alert policy.",
+			BuiltIn:     true,
+			Steps: []RunbookStep{
+				{Title: "Check agent status", Command: "cluster agent status"},
+				{Title: "Check cluster drift", Command: "cluster drift"},
+				{Title: "Check VM states", Command: "cluster alert-vm check"},
+				{Title: "Inspect VM allocations", Command: "kvm top"},
+			},
+		},
+		{
+			ID:          "traffic-billing-audit",
+			Name:        "Traffic Billing Audit",
+			Description: "Collect interface P95 and graph evidence for billing period.",
+			BuiltIn:     true,
+			Steps: []RunbookStep{
+				{Title: "List interfaces", Command: "cluster usage --range 1h"},
+				{Title: "Compute P95 for target iface", Command: "cluster p95 --iface  --range 30d --graph"},
+				{Title: "Export report", Command: "cluster report export --format json --out ./report.json"},
+			},
+		},
+	}
+}
+
+func (s *Service) runbookPath() string {
+	return filepath.Join(s.DataDir(), "runbooks", "custom.json")
+}
+
+func (s *Service) loadCustomRunbooks() ([]Runbook, error) {
+	path := s.runbookPath()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []Runbook{}, nil
+		}
+		return nil, err
+	}
+	if len(strings.TrimSpace(string(raw))) == 0 {
+		return []Runbook{}, nil
+	}
+	var items []Runbook
+	if err := json.Unmarshal(raw, &items); err != nil {
+		return nil, err
+	}
+	out := make([]Runbook, 0, len(items))
+	for _, rb := range items {
+		rb.ID = strings.TrimSpace(rb.ID)
+		rb.Name = strings.TrimSpace(rb.Name)
+		if rb.ID == "" || rb.Name == "" {
+			continue
+		}
+		rb.BuiltIn = false
+		out = append(out, rb)
+	}
+	return out, nil
+}
+
+func (s *Service) saveCustomRunbooks(items []Runbook) error {
+	path := s.runbookPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	b, err := json.MarshalIndent(items, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+func (s *Service) ListRunbooks() ([]Runbook, error) {
+	built := builtinRunbooks()
+	custom, err := s.loadCustomRunbooks()
+	if err != nil {
+		return nil, err
+	}
+	all := append(append([]Runbook{}, built...), custom...)
+	sort.Slice(all, func(i, j int) bool { return strings.ToLower(all[i].ID) < strings.ToLower(all[j].ID) })
+	return all, nil
+}
+
+func (s *Service) GetRunbook(selector string) (Runbook, bool) {
+	selector = strings.TrimSpace(selector)
+	items, err := s.ListRunbooks()
+	if err != nil {
+		return Runbook{}, false
+	}
+	for _, rb := range items {
+		if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
+			return rb, true
+		}
+	}
+	return Runbook{}, false
+}
+
+func (s *Service) AddRunbook(rb Runbook) (Runbook, error) {
+	rb.ID = strings.TrimSpace(rb.ID)
+	rb.Name = strings.TrimSpace(rb.Name)
+	if rb.ID == "" || rb.Name == "" {
+		return Runbook{}, errors.New("runbook id and name are required")
+	}
+	if len(rb.Steps) == 0 {
+		return Runbook{}, errors.New("runbook steps are required")
+	}
+	if b, ok := s.GetRunbook(rb.ID); ok && b.BuiltIn {
+		return Runbook{}, errors.New("cannot overwrite built-in runbook")
+	}
+	custom, err := s.loadCustomRunbooks()
+	if err != nil {
+		return Runbook{}, err
+	}
+	for _, it := range custom {
+		if strings.EqualFold(it.ID, rb.ID) {
+			return Runbook{}, errors.New("runbook id already exists")
+		}
+	}
+	now := s.now().UTC()
+	rb.BuiltIn = false
+	rb.CreatedAt = now
+	rb.UpdatedAt = now
+	custom = append(custom, rb)
+	if err := s.saveCustomRunbooks(custom); err != nil {
+		return Runbook{}, err
+	}
+	_ = s.AppendChange("runbook.add", rb.ID, rb.Name)
+	return rb, nil
+}
+
+func (s *Service) RemoveRunbook(selector string) (Runbook, error) {
+	custom, err := s.loadCustomRunbooks()
+	if err != nil {
+		return Runbook{}, err
+	}
+	idx := -1
+	for i, rb := range custom {
+		if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
+			idx = i
+			break
+		}
+	}
+	if idx < 0 {
+		return Runbook{}, errors.New("custom runbook not found")
+	}
+	removed := custom[idx]
+	custom = append(custom[:idx], custom[idx+1:]...)
+	if err := s.saveCustomRunbooks(custom); err != nil {
+		return Runbook{}, err
+	}
+	_ = s.AppendChange("runbook.remove", removed.ID, removed.Name)
+	return removed, nil
+}
diff --git a/internal/cluster/scheduler.go b/internal/cluster/scheduler.go
new file mode 100644
index 0000000..190406b
--- /dev/null
+++ b/internal/cluster/scheduler.go
@@ -0,0 +1,301 @@
+package cluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+type ScheduledTask struct {
+	ID        string    `json:"id"`
+	Name      string    `json:"name"`
+	Cluster   string    `json:"cluster,omitempty"`
+	Command   string    `json:"command"`
+	Mode      string    `json:"mode,omitempty"` // shell|observer
+	Every     string    `json:"every"`
+	Backoff   string    `json:"backoff,omitempty"`
+	JitterSec int       `json:"jitter_sec,omitempty"`
+	RetryMax  int       `json:"retry_max,omitempty"`
+	RetryCur  int       `json:"retry_cur,omitempty"`
+	Enabled   bool      `json:"enabled"`
+	CreatedAt time.Time `json:"created_at"`
+	UpdatedAt time.Time `json:"updated_at"`
+	LastRunAt time.Time `json:"last_run_at,omitempty"`
+	NextRunAt time.Time `json:"next_run_at,omitempty"`
+}
+
+type SchedulerRunResult struct {
+	Task     ScheduledTask `json:"task"`
+	Ran      bool          `json:"ran"`
+	Error    string        `json:"error,omitempty"`
+	Output   string        `json:"output,omitempty"`
+	ExitCode int           `json:"exit_code,omitempty"`
+}
+
+func (s *Service) schedulerPath() string {
+	return filepath.Join(s.DataDir(), "scheduler", "tasks.json")
+}
+
+func (s *Service) loadTasks() ([]ScheduledTask, error) {
+	path := s.schedulerPath()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []ScheduledTask{}, nil
+		}
+		return nil, err
+	}
+	var items []ScheduledTask
+	if len(strings.TrimSpace(string(raw))) == 0 {
+		return []ScheduledTask{}, nil
+	}
+	if err := json.Unmarshal(raw, &items); err != nil {
+		return nil, err
+	}
+	for i := range items {
+		items[i] = normalizeScheduledTask(items[i])
+	}
+	sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
+	return items, nil
+}
+
+func (s *Service) saveTasks(items []ScheduledTask) error {
+	path := s.schedulerPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	b, err := json.MarshalIndent(items, "", "  ")
+	if err != nil {
+		return err
+	}
+	b = append(b, '\n')
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, b, 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+func normalizeScheduledTask(t ScheduledTask) ScheduledTask {
+	t.ID = strings.TrimSpace(t.ID)
+	t.Name = strings.TrimSpace(t.Name)
+	t.Command = strings.TrimSpace(t.Command)
+	t.Cluster = strings.TrimSpace(t.Cluster)
+	t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
+	t.Every = strings.TrimSpace(t.Every)
+	if t.ID == "" {
+		t.ID = newClusterID()
+	}
+	if t.Name == "" {
+		t.Name = t.ID
+	}
+	if t.Every == "" {
+		t.Every = "5m"
+	}
+	if strings.TrimSpace(t.Backoff) == "" {
+		t.Backoff = "30s"
+	}
+	if t.JitterSec < 0 {
+		t.JitterSec = 0
+	}
+	if t.RetryMax <= 0 {
+		t.RetryMax = 3
+	}
+	if t.RetryCur < 0 {
+		t.RetryCur = 0
+	}
+	if t.Mode == "" {
+		t.Mode = "shell"
+	}
+	if t.Mode != "shell" && t.Mode != "observer" {
+		t.Mode = "shell"
+	}
+	if t.CreatedAt.IsZero() {
+		t.CreatedAt = time.Now().UTC()
+	}
+	if t.UpdatedAt.IsZero() {
+		t.UpdatedAt = t.CreatedAt
+	}
+	return t
+}
+
+func parseTaskEvery(v string) (time.Duration, error) {
+	d, err := time.ParseDuration(strings.TrimSpace(v))
+	if err != nil {
+		return 0, fmt.Errorf("invalid --every duration: %w", err)
+	}
+	if d < time.Minute {
+		return 0, errors.New("--every must be >= 1m")
+	}
+	return d, nil
+}
+
+func (s *Service) SchedulerList() ([]ScheduledTask, error) {
+	return s.loadTasks()
+}
+
+func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) {
+	name = strings.TrimSpace(name)
+	command = strings.TrimSpace(command)
+	every = strings.TrimSpace(every)
+	mode = strings.ToLower(strings.TrimSpace(mode))
+	if name == "" {
+		return ScheduledTask{}, errors.New("task name is required")
+	}
+	if command == "" {
+		return ScheduledTask{}, errors.New("task command is required")
+	}
+	if mode == "" {
+		mode = "shell"
+	}
+	if mode != "shell" && mode != "observer" {
+		return ScheduledTask{}, errors.New("task mode must be shell|observer")
+	}
+	d, err := parseTaskEvery(every)
+	if err != nil {
+		return ScheduledTask{}, err
+	}
+	if strings.TrimSpace(backoff) == "" {
+		backoff = "30s"
+	}
+	if _, err := time.ParseDuration(backoff); err != nil {
+		return ScheduledTask{}, errors.New("invalid backoff duration")
+	}
+	if jitterSec < 0 {
+		jitterSec = 0
+	}
+	if retryMax <= 0 {
+		retryMax = 3
+	}
+	items, err := s.loadTasks()
+	if err != nil {
+		return ScheduledTask{}, err
+	}
+	for _, t := range items {
+		if strings.EqualFold(t.Name, name) {
+			return ScheduledTask{}, fmt.Errorf("task %q already exists", name)
+		}
+	}
+	now := s.now().UTC()
+	t := ScheduledTask{
+		ID:        newClusterID(),
+		Name:      name,
+		Cluster:   strings.TrimSpace(clusterSel),
+		Command:   command,
+		Mode:      mode,
+		Every:     every,
+		Backoff:   backoff,
+		JitterSec: jitterSec,
+		RetryMax:  retryMax,
+		Enabled:   enabled,
+		CreatedAt: now,
+		UpdatedAt: now,
+	}
+	if enabled {
+		t.NextRunAt = now.Add(d)
+	}
+	items = append(items, t)
+	if err := s.saveTasks(items); err != nil {
+		return ScheduledTask{}, err
+	}
+	_ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every))
+	return t, nil
+}
+
+func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) {
+	items, err := s.loadTasks()
+	if err != nil {
+		return ScheduledTask{}, err
+	}
+	selector = strings.TrimSpace(selector)
+	if selector == "" {
+		return ScheduledTask{}, errors.New("task name or id is required")
+	}
+	idx := -1
+	for i, t := range items {
+		if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
+			idx = i
+			break
+		}
+	}
+	if idx < 0 {
+		return ScheduledTask{}, errors.New("task not found")
+	}
+	removed := items[idx]
+	items = append(items[:idx], items[idx+1:]...)
+	if err := s.saveTasks(items); err != nil {
+		return ScheduledTask{}, err
+	}
+	_ = s.AppendChange("scheduler.remove", removed.Name, removed.Command)
+	return removed, nil
+}
+
+func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error {
+	items, err := s.loadTasks()
+	if err != nil {
+		return err
+	}
+	for i := range items {
+		if items[i].ID != taskID {
+			continue
+		}
+		items[i].LastRunAt = ranAt.UTC()
+		if success {
+			d, err := parseTaskEvery(items[i].Every)
+			if err != nil {
+				return err
+			}
+			items[i].RetryCur = 0
+			items[i].NextRunAt = items[i].LastRunAt.Add(d)
+		} else {
+			items[i].RetryCur++
+			if items[i].RetryCur > items[i].RetryMax {
+				// cap retries and move to the next normal run window
+				items[i].RetryCur = 0
+				d, err := parseTaskEvery(items[i].Every)
+				if err != nil {
+					return err
+				}
+				items[i].NextRunAt = items[i].LastRunAt.Add(d)
+			} else {
+				back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff))
+				if err != nil || back <= 0 {
+					back = 30 * time.Second
+				}
+				delay := back * time.Duration(1<<(items[i].RetryCur-1))
+				if items[i].JitterSec > 0 {
+					delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second
+				}
+				items[i].NextRunAt = items[i].LastRunAt.Add(delay)
+			}
+		}
+		items[i].UpdatedAt = ranAt.UTC()
+		break
+	}
+	return s.saveTasks(items)
+}
+
+func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) {
+	items, err := s.loadTasks()
+	if err != nil {
+		return nil, err
+	}
+	n := now.UTC()
+	due := make([]ScheduledTask, 0)
+	for _, t := range items {
+		if !t.Enabled {
+			continue
+		}
+		if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) {
+			due = append(due, t)
+		}
+	}
+	sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) })
+	return due, nil
+}
diff --git a/internal/cluster/service.go b/internal/cluster/service.go
new file mode 100644
index 0000000..836d8f7
--- /dev/null
+++ b/internal/cluster/service.go
@@ -0,0 +1,2602 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"math/big"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/user"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/knownhosts"
+
+	"pxmon/internal/agent"
+	"pxmon/internal/history"
+)
+
+const (
+	defaultSSHPort     = 22
+	defaultProbeTO     = 8 * time.Second
+	defaultAgentPort   = 19090
+	defaultAgentListen = "0.0.0.0:19090"
+)
+
+var (
+	ErrClusterNotFound = errors.New("cluster not found")
+	ErrNoActiveCluster = errors.New("no active cluster selected")
+)
+
+// Service manages node inventory and SSH/agent operations.
+type Service struct {
+	store *Store
+	now   func() time.Time
+
+	tunnelMu   sync.Mutex
+	tunnelPool map[string]*tunneledSSH
+
+	networkStore *history.NetworkStore
+}
+
+// AttachNetworkStore wires a persistent network history store to the service
+// so non-TUI callers (bot, CLI one-shots) can read the same data the TUI
+// monitor writes.
+func (s *Service) AttachNetworkStore(store *history.NetworkStore) {
+	if s == nil {
+		return
+	}
+	s.networkStore = store
+}
+
+// NetworkStore returns the attached network history store or nil.
+func (s *Service) NetworkStore() *history.NetworkStore {
+	if s == nil {
+		return nil
+	}
+	return s.networkStore
+}
+
+// tunneledSSH caches an ssh.Client used to tunnel agent HTTP traffic for an
+// ipfabric node. Reused across probes so we don't pay a new SSH handshake per
+// HTTP call.
+type tunneledSSH struct {
+	client *ssh.Client
+	fp     string // credential fingerprint; lets us invalidate on re-auth
+}
+
+func NewService(store *Store) *Service {
+	return &Service{
+		store:      store,
+		now:        time.Now,
+		tunnelPool: make(map[string]*tunneledSSH),
+	}
+}
+
+func (s *Service) DataDir() string {
+	if s == nil || s.store == nil {
+		return "."
+	}
+	return filepath.Dir(s.store.Path())
+}
+
+func (s *Service) ConfigPath() string {
+	if s == nil || s.store == nil {
+		return ""
+	}
+	return s.store.Path()
+}
+
+// ConnectOptions contains SSH node registration parameters.
+type ConnectOptions struct {
+	Name                   string
+	Host                   string
+	Port                   int
+	User                   string
+	Transport              TransportMode
+	AuthMethod             AuthMethod
+	Password               string
+	StorePassword          bool
+	KeyPath                string
+	KeyPassphrase          string
+	KeyPassphraseFile      string
+	StoreKeyPassphrase     bool
+	StoreKeyPassphraseFile bool
+	InsecureHostKey        bool
+	SkipCheck              bool
+	AllowUnreachable       bool
+	Force                  bool
+}
+
+// SSHProbeResult describes SSH connectivity status.
+type SSHProbeResult struct {
+	Reachable bool      `json:"reachable"`
+	Address   string    `json:"address"`
+	LatencyMS int64     `json:"latency_ms"`
+	Error     string    `json:"error,omitempty"`
+	CheckedAt time.Time `json:"checked_at"`
+}
+
+// AgentPingResult describes pxmon-agent availability.
+type AgentPingResult struct {
+	Reachable  bool      `json:"reachable"`
+	Endpoint   string    `json:"endpoint"`
+	StatusCode int       `json:"status_code"`
+	Version    string    `json:"version,omitempty"`
+	Error      string    `json:"error,omitempty"`
+	CheckedAt  time.Time `json:"checked_at"`
+}
+
+// BootstrapOptions configures agent installation over SSH.
+type BootstrapOptions struct {
+	Selector        string
+	Password        string
+	KeyPassphrase   string
+	ListenAddress   string
+	AgentPort       int
+	LocalAgentBin   string
+	RotateToken     bool
+	AllowAgentProbe bool
+}
+
+// BootstrapResult returns details about deployed agent.
+type BootstrapResult struct {
+	RemoteOS   string          `json:"remote_os"`
+	RemoteArch string          `json:"remote_arch"`
+	PID        string          `json:"pid"`
+	AgentPing  AgentPingResult `json:"agent_ping"`
+}
+
+func (s *Service) Connect(ctx context.Context, opts ConnectOptions) (Cluster, SSHProbeResult, error) {
+	name := strings.TrimSpace(opts.Name)
+	host := strings.TrimSpace(opts.Host)
+	if name == "" {
+		return Cluster{}, SSHProbeResult{}, errors.New("--name is required")
+	}
+	if host == "" {
+		return Cluster{}, SSHProbeResult{}, errors.New("--host is required")
+	}
+	if strings.Contains(host, "://") {
+		return Cluster{}, SSHProbeResult{}, errors.New("--host must be hostname or IP, not URL")
+	}
+
+	port := opts.Port
+	if port == 0 {
+		port = defaultSSHPort
+	}
+	if port < 1 || port > 65535 {
+		return Cluster{}, SSHProbeResult{}, errors.New("--port must be in range 1..65535")
+	}
+
+	username := strings.TrimSpace(opts.User)
+	if username == "" {
+		if u, err := user.Current(); err == nil {
+			username = u.Username
+		}
+	}
+	if username == "" {
+		return Cluster{}, SSHProbeResult{}, errors.New("--user is required")
+	}
+
+	authMethod := opts.AuthMethod
+	if authMethod == "" {
+		authMethod = AuthMethodKey
+	}
+	if authMethod != AuthMethodPassword && authMethod != AuthMethodKey {
+		return Cluster{}, SSHProbeResult{}, fmt.Errorf("unsupported auth method %q", authMethod)
+	}
+
+	keyPath := ""
+	if authMethod == AuthMethodKey {
+		if strings.TrimSpace(opts.KeyPath) == "" {
+			return Cluster{}, SSHProbeResult{}, errors.New("--key-path is required for key auth")
+		}
+		expanded, err := expandPath(opts.KeyPath)
+		if err != nil {
+			return Cluster{}, SSHProbeResult{}, err
+		}
+		if _, err := os.Stat(expanded); err != nil {
+			return Cluster{}, SSHProbeResult{}, fmt.Errorf("read key file: %w", err)
+		}
+		keyPath = expanded
+	}
+	if authMethod == AuthMethodPassword && strings.TrimSpace(opts.Password) == "" {
+		return Cluster{}, SSHProbeResult{}, errors.New("--password is required for password auth")
+	}
+
+	storedPassword := ""
+	if opts.StorePassword {
+		storedPassword = opts.Password
+	}
+	storedPassphrase := ""
+	if opts.StoreKeyPassphrase {
+		storedPassphrase = opts.KeyPassphrase
+	}
+	storedPassphraseFile := ""
+	if opts.StoreKeyPassphraseFile {
+		expanded, err := expandPath(opts.KeyPassphraseFile)
+		if err != nil {
+			return Cluster{}, SSHProbeResult{}, err
+		}
+		if _, err := os.Stat(expanded); err != nil {
+			return Cluster{}, SSHProbeResult{}, fmt.Errorf("read key passphrase file: %w", err)
+		}
+		storedPassphraseFile = expanded
+	}
+
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, SSHProbeResult{}, err
+	}
+
+	now := s.now().UTC()
+	idxByName := -1
+	for i, c := range reg.Clusters {
+		if strings.EqualFold(c.Name, name) {
+			idxByName = i
+			break
+		}
+	}
+
+	candidate := Cluster{
+		ID:                newClusterID(),
+		Name:              name,
+		Host:              host,
+		Port:              port,
+		User:              username,
+		Transport:         normalizeTransport(opts.Transport),
+		AuthMethod:        authMethod,
+		Password:          storedPassword,
+		KeyPath:           keyPath,
+		KeyPassphrase:     storedPassphrase,
+		KeyPassphraseFile: storedPassphraseFile,
+		InsecureHostKey:   opts.InsecureHostKey,
+		Alerts:            defaultAlertPolicy(),
+		CreatedAt:         now,
+		UpdatedAt:         now,
+	}
+
+	var probe SSHProbeResult
+	if !opts.SkipCheck {
+		probe = s.probeSSH(ctx, candidate, opts.Password, opts.KeyPassphrase)
+		if !probe.Reachable && !opts.AllowUnreachable {
+			return Cluster{}, probe, fmt.Errorf("ssh probe failed, use --allow-unreachable to save anyway: %s", probe.Error)
+		}
+		if probe.Reachable {
+			if software, softErr := s.probeSoftware(ctx, candidate, opts.Password, opts.KeyPassphrase); softErr == nil {
+				candidate.Software = software
+			}
+		}
+	}
+
+	if idxByName >= 0 {
+		if !opts.Force {
+			return Cluster{}, probe, fmt.Errorf("cluster with name %q already exists, use --force to overwrite", name)
+		}
+
+		existing := reg.Clusters[idxByName]
+		existing.Host = candidate.Host
+		existing.Port = candidate.Port
+		existing.User = candidate.User
+		existing.Transport = candidate.Transport
+		existing.AuthMethod = candidate.AuthMethod
+		existing.Password = candidate.Password
+		existing.KeyPath = candidate.KeyPath
+		existing.KeyPassphrase = candidate.KeyPassphrase
+		existing.KeyPassphraseFile = candidate.KeyPassphraseFile
+		existing.InsecureHostKey = candidate.InsecureHostKey
+		existing.Alerts = ensureAlertPolicy(existing.Alerts)
+		if !candidate.Software.DetectedAt.IsZero() {
+			existing.Software = candidate.Software
+		}
+		existing.UpdatedAt = now
+
+		reg.Clusters[idxByName] = existing
+		reg.ActiveClusterID = existing.ID
+		if err := s.store.Save(reg); err != nil {
+			return Cluster{}, probe, err
+		}
+		return existing, probe, nil
+	}
+
+	if existsTarget(reg.Clusters, candidate.Host, candidate.Port, candidate.User) {
+		return Cluster{}, probe, fmt.Errorf("cluster with target %s@%s:%d already exists", candidate.User, candidate.Host, candidate.Port)
+	}
+
+	reg.Clusters = append(reg.Clusters, candidate)
+	reg.ActiveClusterID = candidate.ID
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, probe, err
+	}
+
+	return candidate, probe, nil
+}
+
+func (s *Service) GetAlertPolicy(selector string) (AlertPolicy, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return AlertPolicy{}, err
+	}
+	return ensureAlertPolicy(c.Alerts), nil
+}
+
+func (s *Service) SetAlertPolicy(selector string, policy AlertPolicy) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	cluster, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	cluster.Alerts = ensureAlertPolicy(policy)
+	cluster.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = cluster
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+
+	return cluster, nil
+}
+
+func (s *Service) GetTelegram() (Telegram, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Telegram{}, err
+	}
+	return normalizeTelegram(reg.Telegram), nil
+}
+
+func (s *Service) SetTelegram(cfg Telegram) (Telegram, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Telegram{}, err
+	}
+
+	normalized, err := validateTelegramInput(cfg)
+	if err != nil {
+		return Telegram{}, err
+	}
+	normalized.UpdatedAt = s.now().UTC()
+	reg.Telegram = normalized
+
+	if err := s.store.Save(reg); err != nil {
+		return Telegram{}, err
+	}
+	return normalized, nil
+}
+
+func (s *Service) DisableTelegram() (Telegram, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Telegram{}, err
+	}
+
+	cfg := normalizeTelegram(reg.Telegram)
+	cfg.Enabled = false
+	cfg.UpdatedAt = s.now().UTC()
+	reg.Telegram = cfg
+
+	if err := s.store.Save(reg); err != nil {
+		return Telegram{}, err
+	}
+	return cfg, nil
+}
+
+func (s *Service) GetLocker() (Locker, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Locker{}, err
+	}
+	return normalizeLocker(reg.Locker), nil
+}
+
+func (s *Service) AuditLocker(event, detail string) {
+	if s == nil || s.store == nil {
+		return
+	}
+	_ = s.store.AppendLockerAudit(event, detail)
+}
+
+func (s *Service) SetLockerPassword(password string) (Locker, error) {
+	pass := strings.TrimSpace(password)
+	if len(pass) < 4 {
+		s.AuditLocker("locker_set_password_failed", "password too short")
+		return Locker{}, errors.New("locker password must be at least 4 characters")
+	}
+	hash, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost)
+	if err != nil {
+		s.AuditLocker("locker_set_password_failed", "bcrypt error")
+		return Locker{}, err
+	}
+
+	reg, err := s.store.Load()
+	if err != nil {
+		return Locker{}, err
+	}
+	cfg := normalizeLocker(reg.Locker)
+	cfg.PasswordHash = string(hash)
+	cfg.Enabled = true
+	cfg.UpdatedAt = s.now().UTC()
+	reg.Locker = cfg
+	if err := s.store.Save(reg); err != nil {
+		s.AuditLocker("locker_set_password_failed", "save registry failed")
+		return Locker{}, err
+	}
+	_ = s.store.ClearLockerSession()
+	if err := s.store.SaveLockerSession(cfg.PasswordHash, 6*time.Hour); err != nil {
+		s.AuditLocker("locker_set_password_failed", "save locker session failed")
+		return Locker{}, err
+	}
+	s.AuditLocker("locker_set_password", "locker enabled and session issued")
+	return cfg, nil
+}
+
+func (s *Service) SetLockerEnabled(enabled bool) (Locker, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Locker{}, err
+	}
+	cfg := normalizeLocker(reg.Locker)
+	if enabled && strings.TrimSpace(cfg.PasswordHash) == "" {
+		s.AuditLocker("locker_set_enabled_failed", "password not set")
+		return Locker{}, errors.New("locker password is not set")
+	}
+	cfg.Enabled = enabled
+	cfg.UpdatedAt = s.now().UTC()
+	reg.Locker = cfg
+	if err := s.store.Save(reg); err != nil {
+		s.AuditLocker("locker_set_enabled_failed", "save registry failed")
+		return Locker{}, err
+	}
+	if !enabled {
+		_ = s.store.ClearLockerSession()
+		s.AuditLocker("locker_disabled", "session cleared")
+	} else {
+		s.AuditLocker("locker_enabled", "locker enabled")
+	}
+	return cfg, nil
+}
+
+func (s *Service) UnlockLocker(password string) error {
+	cfg, err := s.GetLocker()
+	if err != nil {
+		s.AuditLocker("locker_unlock_failed", "load locker config failed")
+		return err
+	}
+	if !cfg.Enabled {
+		return nil
+	}
+	if strings.TrimSpace(cfg.PasswordHash) == "" {
+		s.AuditLocker("locker_unlock_failed", "password hash not set")
+		return errors.New("locker password is not set")
+	}
+	if err := bcrypt.CompareHashAndPassword([]byte(cfg.PasswordHash), []byte(password)); err != nil {
+		s.AuditLocker("locker_unlock_failed", "invalid password")
+		return errors.New("invalid locker password")
+	}
+	if err := s.store.SaveLockerSession(cfg.PasswordHash, 6*time.Hour); err != nil {
+		s.AuditLocker("locker_unlock_failed", "save session failed")
+		return err
+	}
+	s.AuditLocker("locker_unlocked", "session issued for 6h")
+	return nil
+}
+
+func (s *Service) LockNow() error {
+	if err := s.store.ClearLockerSession(); err != nil {
+		s.AuditLocker("locker_lock_now_failed", "clear session failed")
+		return err
+	}
+	s.AuditLocker("locker_locked", "session cleared")
+	return nil
+}
+
+func (s *Service) IsLocked() (bool, time.Time, error) {
+	cfg, err := s.GetLocker()
+	if err != nil {
+		return true, time.Time{}, err
+	}
+	if !cfg.Enabled {
+		return false, time.Time{}, nil
+	}
+	ok, expiresAt, err := s.store.ValidateLockerSession(cfg.PasswordHash)
+	if err != nil {
+		return true, time.Time{}, err
+	}
+	return !ok, expiresAt, nil
+}
+
+func (s *Service) List() ([]Cluster, string, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return nil, "", err
+	}
+
+	clusters := append([]Cluster(nil), reg.Clusters...)
+	sort.Slice(clusters, func(i, j int) bool {
+		return strings.ToLower(clusters[i].Name) < strings.ToLower(clusters[j].Name)
+	})
+
+	return clusters, reg.ActiveClusterID, nil
+}
+
+func normalizeTelegram(cfg Telegram) Telegram {
+	cfg.Token = strings.TrimSpace(cfg.Token)
+	cfg.AllowedUserIDs = normalizeUserIDs(cfg.AllowedUserIDs)
+	if cfg.Token == "" || len(cfg.AllowedUserIDs) == 0 {
+		cfg.Enabled = false
+	}
+	return cfg
+}
+
+func validateTelegramInput(cfg Telegram) (Telegram, error) {
+	cfg = normalizeTelegram(cfg)
+	if cfg.Token == "" {
+		return Telegram{}, errors.New("telegram token is required")
+	}
+	if len(cfg.AllowedUserIDs) == 0 {
+		return Telegram{}, errors.New("at least one telegram user id is required")
+	}
+	if cfg.Enabled && cfg.Token == "" {
+		return Telegram{}, errors.New("telegram cannot be enabled without token")
+	}
+	return cfg, nil
+}
+
+func normalizeUserIDs(ids []int64) []int64 {
+	if len(ids) == 0 {
+		return nil
+	}
+	seen := make(map[int64]struct{}, len(ids))
+	out := make([]int64, 0, len(ids))
+	for _, id := range ids {
+		if id <= 0 {
+			continue
+		}
+		if _, ok := seen[id]; ok {
+			continue
+		}
+		seen[id] = struct{}{}
+		out = append(out, id)
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i] < out[j] })
+	return out
+}
+
+func (s *Service) Current() (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	if reg.ActiveClusterID == "" {
+		return Cluster{}, ErrNoActiveCluster
+	}
+
+	cluster, _, err := findCluster(reg, reg.ActiveClusterID)
+	if err != nil {
+		return Cluster{}, ErrNoActiveCluster
+	}
+	return cluster, nil
+}
+
+func (s *Service) Get(selector string) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	cluster, _, err := findCluster(reg, selector)
+	return cluster, err
+}
+
+func (s *Service) Use(selector string) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	cluster, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	reg.ActiveClusterID = reg.Clusters[idx].ID
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+
+	return cluster, nil
+}
+
+func (s *Service) Disconnect(selector string) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	cluster, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	reg.Clusters = append(reg.Clusters[:idx], reg.Clusters[idx+1:]...)
+	if reg.ActiveClusterID == cluster.ID {
+		reg.ActiveClusterID = ""
+		if len(reg.Clusters) > 0 {
+			reg.ActiveClusterID = reg.Clusters[0].ID
+		}
+	}
+
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+
+	s.CloseTunnelClient(cluster.ID)
+	return cluster, nil
+}
+
+func (s *Service) ProbeSSH(ctx context.Context, selector, password, keyPassphrase string) (SSHProbeResult, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return SSHProbeResult{}, err
+	}
+	return s.probeSSH(ctx, cluster, password, keyPassphrase), nil
+}
+
+// UpdateAuthOptions is a partial update of a cluster's authentication settings.
+// Nil pointer fields mean "leave as-is"; non-nil means "replace with this value".
+type UpdateAuthOptions struct {
+	AuthMethod             AuthMethod
+	SetPassword            bool
+	Password               string
+	StorePassword          bool
+	ClearPassword          bool
+	SetKeyPath             bool
+	KeyPath                string
+	SetKeyPassphrase       bool
+	KeyPassphrase          string
+	StoreKeyPassphrase     bool
+	SetKeyPassphraseFile   bool
+	KeyPassphraseFile      string
+	StoreKeyPassphraseFile bool
+	ClearKeyPassphrase     bool
+	SetInsecureHostKey     bool
+	InsecureHostKey        bool
+	SetTransport           bool
+	Transport              TransportMode
+}
+
+// UpdateAuth mutates stored credentials for a cluster and persists.
+func (s *Service) UpdateAuth(selector string, opts UpdateAuthOptions) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+
+	if opts.AuthMethod != "" {
+		if opts.AuthMethod != AuthMethodPassword && opts.AuthMethod != AuthMethodKey {
+			return Cluster{}, fmt.Errorf("unsupported auth method %q", opts.AuthMethod)
+		}
+		c.AuthMethod = opts.AuthMethod
+	}
+
+	if opts.ClearPassword {
+		c.Password = ""
+	} else if opts.SetPassword {
+		if opts.StorePassword {
+			c.Password = opts.Password
+		} else {
+			c.Password = ""
+		}
+	}
+
+	if opts.SetKeyPath {
+		if strings.TrimSpace(opts.KeyPath) == "" {
+			c.KeyPath = ""
+		} else {
+			expanded, err := expandPath(opts.KeyPath)
+			if err != nil {
+				return Cluster{}, err
+			}
+			if _, err := os.Stat(expanded); err != nil {
+				return Cluster{}, fmt.Errorf("read key file: %w", err)
+			}
+			c.KeyPath = expanded
+		}
+	}
+
+	if opts.ClearKeyPassphrase {
+		c.KeyPassphrase = ""
+		c.KeyPassphraseFile = ""
+	} else if opts.SetKeyPassphrase {
+		if opts.StoreKeyPassphrase {
+			c.KeyPassphrase = opts.KeyPassphrase
+		} else {
+			c.KeyPassphrase = ""
+		}
+	}
+	if opts.SetKeyPassphraseFile {
+		if opts.StoreKeyPassphraseFile {
+			expanded, err := expandPath(opts.KeyPassphraseFile)
+			if err != nil {
+				return Cluster{}, err
+			}
+			if _, err := os.Stat(expanded); err != nil {
+				return Cluster{}, fmt.Errorf("read key passphrase file: %w", err)
+			}
+			c.KeyPassphraseFile = expanded
+		} else {
+			c.KeyPassphraseFile = ""
+		}
+	}
+
+	if opts.SetInsecureHostKey {
+		c.InsecureHostKey = opts.InsecureHostKey
+	}
+
+	if opts.SetTransport {
+		t := normalizeTransport(opts.Transport)
+		if t != TransportDirect && t != TransportIPFabric {
+			return Cluster{}, fmt.Errorf("unsupported transport %q", opts.Transport)
+		}
+		c.Transport = t
+	}
+
+	// Sanity: password auth needs a password when used live; allow empty if
+	// the caller is only flipping method and will provide password later.
+	switch c.AuthMethod {
+	case AuthMethodKey:
+		if strings.TrimSpace(c.KeyPath) == "" {
+			return Cluster{}, errors.New("key auth requires key_path; set it with --key-path")
+		}
+	case AuthMethodPassword:
+		// Password may be stored or supplied at runtime; no hard check here.
+	default:
+		return Cluster{}, fmt.Errorf("unsupported auth method %q", c.AuthMethod)
+	}
+
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	s.CloseTunnelClient(c.ID)
+	return c, nil
+}
+
+// InteractiveShellOptions controls an interactive SSH session.
+type InteractiveShellOptions struct {
+	Stdin   io.Reader
+	Stdout  io.Writer
+	Stderr  io.Writer
+	Term    string
+	Width   int
+	Height  int
+	Resize  <-chan InteractiveShellSize
+	Command string // optional remote command; empty means interactive shell
+}
+
+// InteractiveShellSize carries terminal resize events.
+type InteractiveShellSize struct {
+	Width  int
+	Height int
+}
+
+// OpenInteractiveShell opens an interactive SSH session against the selected
+// cluster using stored credentials. The caller is responsible for putting its
+// local stdin into raw mode and delivering SIGWINCH events through opts.Resize.
+func (s *Service) OpenInteractiveShell(ctx context.Context, selector string, opts InteractiveShellOptions) error {
+	c, err := s.Get(selector)
+	if err != nil {
+		return err
+	}
+
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return fmt.Errorf("ssh dial: %w", err)
+	}
+	defer client.Close()
+
+	session, err := client.NewSession()
+	if err != nil {
+		return fmt.Errorf("ssh session: %w", err)
+	}
+	defer session.Close()
+
+	termName := strings.TrimSpace(opts.Term)
+	if termName == "" {
+		termName = os.Getenv("TERM")
+	}
+	if termName == "" {
+		termName = "xterm-256color"
+	}
+
+	width := opts.Width
+	height := opts.Height
+	if width <= 0 {
+		width = 120
+	}
+	if height <= 0 {
+		height = 32
+	}
+
+	modes := ssh.TerminalModes{
+		ssh.ECHO:          1,
+		ssh.ICANON:        1,
+		ssh.ISIG:          1,
+		ssh.ICRNL:         1,
+		ssh.OPOST:         1,
+		ssh.TTY_OP_ISPEED: 38400,
+		ssh.TTY_OP_OSPEED: 38400,
+	}
+
+	if err := session.RequestPty(termName, height, width, modes); err != nil {
+		return fmt.Errorf("ssh pty: %w", err)
+	}
+
+	session.Stdout = opts.Stdout
+	session.Stderr = opts.Stderr
+	stdinPipe, err := session.StdinPipe()
+	if err != nil {
+		return fmt.Errorf("ssh stdin: %w", err)
+	}
+
+	if strings.TrimSpace(opts.Command) == "" {
+		if err := session.Shell(); err != nil {
+			return fmt.Errorf("ssh shell: %w", err)
+		}
+	} else {
+		if err := session.Start(opts.Command); err != nil {
+			return fmt.Errorf("ssh start: %w", err)
+		}
+	}
+
+	doneCtx, cancelDone := context.WithCancel(context.Background())
+	defer cancelDone()
+
+	if opts.Resize != nil {
+		go func() {
+			for {
+				select {
+				case <-doneCtx.Done():
+					return
+				case sz, ok := <-opts.Resize:
+					if !ok {
+						return
+					}
+					if sz.Width <= 0 || sz.Height <= 0 {
+						continue
+					}
+					_ = session.WindowChange(sz.Height, sz.Width)
+				}
+			}
+		}()
+	}
+
+	copyDone := make(chan struct{})
+	if opts.Stdin != nil {
+		go func() {
+			_, _ = io.Copy(stdinPipe, opts.Stdin)
+			_ = stdinPipe.Close()
+			close(copyDone)
+		}()
+	} else {
+		close(copyDone)
+	}
+
+	waitErr := make(chan error, 1)
+	go func() {
+		waitErr <- session.Wait()
+	}()
+
+	select {
+	case <-ctx.Done():
+		_ = session.Signal(ssh.SIGHUP)
+		_ = session.Close()
+		<-waitErr
+		return ctx.Err()
+	case err := <-waitErr:
+		if err != nil {
+			var exitErr *ssh.ExitError
+			if errors.As(err, &exitErr) {
+				return nil
+			}
+			if errors.Is(err, io.EOF) {
+				return nil
+			}
+			return err
+		}
+		return nil
+	}
+}
+
+// SoftwareScan refreshes software/plugin detection for a cluster and persists it.
+func (s *Service) SoftwareScan(ctx context.Context, selector, password, keyPassphrase string) (Cluster, SoftwareInfo, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, SoftwareInfo{}, err
+	}
+
+	cluster, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, SoftwareInfo{}, err
+	}
+
+	info, err := s.probeSoftware(ctx, cluster, password, keyPassphrase)
+	if err != nil {
+		return Cluster{}, SoftwareInfo{}, err
+	}
+
+	cluster.Software = info
+	cluster.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = cluster
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, SoftwareInfo{}, err
+	}
+
+	return cluster, info, nil
+}
+
+// InteractiveSession represents an embedded SSH PTY session owned by the caller.
+type InteractiveSession struct {
+	client    *ssh.Client
+	session   *ssh.Session
+	stdin     io.WriteCloser
+	stdout    io.Reader
+	doneCh    chan error
+	closeOnce sync.Once
+	closeErr  error
+}
+
+// Read reads a chunk of PTY output from the remote session.
+func (s *InteractiveSession) Read(p []byte) (int, error) {
+	if s == nil || s.stdout == nil {
+		return 0, io.EOF
+	}
+	return s.stdout.Read(p)
+}
+
+// Write sends a chunk of input to the remote session's stdin.
+func (s *InteractiveSession) Write(p []byte) (int, error) {
+	if s == nil || s.stdin == nil {
+		return 0, io.ErrClosedPipe
+	}
+	return s.stdin.Write(p)
+}
+
+// Resize notifies the remote side about a new terminal size.
+func (s *InteractiveSession) Resize(cols, rows int) error {
+	if s == nil || s.session == nil {
+		return nil
+	}
+	if cols <= 0 || rows <= 0 {
+		return nil
+	}
+	return s.session.WindowChange(rows, cols)
+}
+
+// Wait returns a channel that is closed when the remote session exits.
+func (s *InteractiveSession) Wait() <-chan error {
+	if s == nil {
+		ch := make(chan error, 1)
+		close(ch)
+		return ch
+	}
+	return s.doneCh
+}
+
+// Close tears down the session and SSH client.
+func (s *InteractiveSession) Close() error {
+	if s == nil {
+		return nil
+	}
+	s.closeOnce.Do(func() {
+		if s.stdin != nil {
+			_ = s.stdin.Close()
+		}
+		if s.session != nil {
+			_ = s.session.Close()
+		}
+		if s.client != nil {
+			s.closeErr = s.client.Close()
+		}
+	})
+	return s.closeErr
+}
+
+// StartInteractiveShell opens an SSH connection with a PTY and returns a
+// handle that the caller can drive: reading remote output, writing input,
+// resizing, and closing. Unlike OpenInteractiveShell, this method does NOT
+// touch os.Stdin/os.Stdout — it's meant for embedding inside the TUI.
+func (s *Service) StartInteractiveShell(ctx context.Context, selector string, cols, rows int, termName string) (*InteractiveSession, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return nil, fmt.Errorf("ssh dial: %w", err)
+	}
+
+	session, err := client.NewSession()
+	if err != nil {
+		client.Close()
+		return nil, fmt.Errorf("ssh session: %w", err)
+	}
+
+	if cols <= 0 {
+		cols = 120
+	}
+	if rows <= 0 {
+		rows = 32
+	}
+	if strings.TrimSpace(termName) == "" {
+		termName = "xterm-256color"
+	}
+
+	modes := ssh.TerminalModes{
+		ssh.ECHO:          1,
+		ssh.ICANON:        1,
+		ssh.ISIG:          1,
+		ssh.ICRNL:         1,
+		ssh.OPOST:         1,
+		ssh.TTY_OP_ISPEED: 38400,
+		ssh.TTY_OP_OSPEED: 38400,
+	}
+
+	if err := session.RequestPty(termName, rows, cols, modes); err != nil {
+		_ = session.Close()
+		client.Close()
+		return nil, fmt.Errorf("ssh pty: %w", err)
+	}
+
+	stdinPipe, err := session.StdinPipe()
+	if err != nil {
+		_ = session.Close()
+		client.Close()
+		return nil, fmt.Errorf("ssh stdin: %w", err)
+	}
+	stdoutPipe, err := session.StdoutPipe()
+	if err != nil {
+		_ = session.Close()
+		client.Close()
+		return nil, fmt.Errorf("ssh stdout: %w", err)
+	}
+	stderrPipe, err := session.StderrPipe()
+	if err != nil {
+		_ = session.Close()
+		client.Close()
+		return nil, fmt.Errorf("ssh stderr: %w", err)
+	}
+
+	combined := newCombinedReader(stdoutPipe, stderrPipe)
+
+	if err := session.Shell(); err != nil {
+		_ = session.Close()
+		client.Close()
+		return nil, fmt.Errorf("ssh shell: %w", err)
+	}
+
+	doneCh := make(chan error, 1)
+	go func() {
+		doneCh <- session.Wait()
+		close(doneCh)
+	}()
+
+	return &InteractiveSession{
+		client:  client,
+		session: session,
+		stdin:   stdinPipe,
+		stdout:  combined,
+		doneCh:  doneCh,
+	}, nil
+}
+
+// combinedReader merges stderr into stdout for a PTY session.
+type combinedReader struct {
+	ch  chan combinedChunk
+	buf []byte
+	err error
+}
+
+type combinedChunk struct {
+	data []byte
+	err  error
+}
+
+func newCombinedReader(streams ...io.Reader) *combinedReader {
+	cr := &combinedReader{ch: make(chan combinedChunk, 8)}
+	var wg sync.WaitGroup
+	for _, r := range streams {
+		if r == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(rd io.Reader) {
+			defer wg.Done()
+			buf := make([]byte, 4096)
+			for {
+				n, err := rd.Read(buf)
+				if n > 0 {
+					cp := make([]byte, n)
+					copy(cp, buf[:n])
+					cr.ch <- combinedChunk{data: cp}
+				}
+				if err != nil {
+					cr.ch <- combinedChunk{err: err}
+					return
+				}
+			}
+		}(r)
+	}
+	go func() {
+		wg.Wait()
+		close(cr.ch)
+	}()
+	return cr
+}
+
+func (r *combinedReader) Read(p []byte) (int, error) {
+	if len(r.buf) > 0 {
+		n := copy(p, r.buf)
+		r.buf = r.buf[n:]
+		return n, nil
+	}
+	if r.err != nil {
+		return 0, r.err
+	}
+	chunk, ok := <-r.ch
+	if !ok {
+		if r.err == nil {
+			r.err = io.EOF
+		}
+		return 0, r.err
+	}
+	if chunk.err != nil {
+		r.err = chunk.err
+		if len(chunk.data) == 0 {
+			return 0, r.err
+		}
+	}
+	n := copy(p, chunk.data)
+	if n < len(chunk.data) {
+		r.buf = chunk.data[n:]
+	}
+	return n, nil
+}
+
+// RunPluginAction executes plugin-specific command over SSH on the selected cluster.
+// Supported tools: kvm, lxc, lxd, bird, frr.
+func (s *Service) RunPluginAction(ctx context.Context, selector, tool, action string, args []string) (string, error) {
+	tool = strings.ToLower(strings.TrimSpace(tool))
+	c, err := s.Get(selector)
+	if err != nil {
+		return "", err
+	}
+
+	supportKnown := softwareProbeKnown(c.Software)
+	if !supportKnown {
+		if refreshed, info, scanErr := s.SoftwareScan(ctx, c.ID, "", ""); scanErr == nil {
+			c = refreshed
+			c.Software = info
+			supportKnown = true
+		}
+	}
+
+	if supportKnown && !isPluginToolSupported(c.Software, tool) {
+		return "", fmt.Errorf("support for %s was not detected on cluster %q. If you think this is a mistake, run `cluster software scan %s` and retry", tool, c.Name, c.Name)
+	}
+
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return "", err
+	}
+	defer client.Close()
+
+	script, err := pluginScript(tool, action, args)
+	if err != nil {
+		return "", err
+	}
+
+	out, err := runRemoteCommand(ctx, client, script)
+	if err != nil {
+		return "", err
+	}
+	return out, nil
+}
+
+func (s *Service) probeSSH(ctx context.Context, c Cluster, password, keyPassphrase string) SSHProbeResult {
+	started := time.Now()
+	res := SSHProbeResult{
+		Reachable: false,
+		Address:   fmt.Sprintf("%s@%s:%d", c.User, c.Host, c.Port),
+		CheckedAt: s.now().UTC(),
+	}
+
+	probeCtx, cancel := context.WithTimeout(ctx, defaultProbeTO)
+	defer cancel()
+
+	client, err := s.dialSSH(probeCtx, c, password, keyPassphrase)
+	if err != nil {
+		res.Error = err.Error()
+		return res
+	}
+	defer client.Close()
+
+	res.Reachable = true
+	res.LatencyMS = time.Since(started).Milliseconds()
+	return res
+}
+
+// RunRemoteShell executes a non-interactive shell command on the selected
+// cluster over SSH and returns combined stdout/stderr text.
+func (s *Service) RunRemoteShell(ctx context.Context, selector, command string) (string, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return "", err
+	}
+	cmd := strings.TrimSpace(command)
+	if cmd == "" {
+		return "", errors.New("empty remote command")
+	}
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return "", err
+	}
+	defer client.Close()
+	return runRemoteCommand(ctx, client, cmd)
+}
+
+func (s *Service) probeSoftware(ctx context.Context, c Cluster, password, keyPassphrase string) (SoftwareInfo, error) {
+	client, err := s.dialSSH(ctx, c, password, keyPassphrase)
+	if err != nil {
+		return SoftwareInfo{}, err
+	}
+	defer client.Close()
+
+	out, err := runRemoteCommand(ctx, client, softwareProbeScript())
+	if err != nil {
+		return SoftwareInfo{}, err
+	}
+	return parseSoftwareProbe(out, s.now().UTC()), nil
+}
+
+func (s *Service) BootstrapAgent(ctx context.Context, opts BootstrapOptions) (Cluster, BootstrapResult, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, err
+	}
+
+	cluster, idx, err := findCluster(reg, opts.Selector)
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, err
+	}
+
+	listenAddress := strings.TrimSpace(opts.ListenAddress)
+	if listenAddress == "" {
+		if normalizeTransport(cluster.Transport) == TransportIPFabric {
+			// ipfabric nodes have no default outbound route; we reach the
+			// agent only through the SSH tunnel, so bind to loopback.
+			listenAddress = "127.0.0.1:19090"
+		} else {
+			listenAddress = defaultAgentListen
+		}
+	}
+
+	agentPort := opts.AgentPort
+	if agentPort == 0 {
+		p, err := parsePortFromListen(listenAddress)
+		if err != nil {
+			return Cluster{}, BootstrapResult{}, err
+		}
+		agentPort = p
+	}
+
+	sshCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+
+	client, err := s.dialSSH(sshCtx, cluster, opts.Password, opts.KeyPassphrase)
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("ssh connect for bootstrap: %w", err)
+	}
+	defer client.Close()
+
+	unameOut, err := runRemoteCommand(sshCtx, client, "uname -s; uname -m")
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("detect remote runtime: %w", err)
+	}
+	lines := splitNonEmptyLines(unameOut)
+	if len(lines) < 2 {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("unexpected uname output: %q", unameOut)
+	}
+
+	remoteOS := strings.TrimSpace(lines[0])
+	remoteArch := strings.TrimSpace(lines[1])
+	goos, goarch, err := mapRuntime(remoteOS, remoteArch)
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, err
+	}
+
+	localBin := strings.TrimSpace(opts.LocalAgentBin)
+	cleanup := func() {}
+	if localBin == "" {
+		var buildErr error
+		localBin, cleanup, buildErr = buildAgentBinary(goos, goarch)
+		if buildErr != nil {
+			return Cluster{}, BootstrapResult{}, buildErr
+		}
+	}
+	defer cleanup()
+
+	if _, err := os.Stat(localBin); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("agent binary not found: %w", err)
+	}
+
+	remoteHome, err := remoteHomeDir(sshCtx, client)
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("resolve remote home: %w", err)
+	}
+	remoteBase := path.Join(remoteHome, ".pxmon")
+	remoteBin := path.Join(remoteBase, "bin", "pxmon-agent")
+	remoteCfg := path.Join(remoteBase, "run", "agent.json")
+	remoteLog := path.Join(remoteBase, "log", "agent.log")
+	remotePID := path.Join(remoteBase, "run", "agent.pid")
+
+	mkdirScript := fmt.Sprintf(
+		"mkdir -p %s %s %s",
+		shellQuote(path.Join(remoteBase, "bin")),
+		shellQuote(path.Join(remoteBase, "run")),
+		shellQuote(path.Join(remoteBase, "log")),
+	)
+	if _, err := runRemoteCommand(sshCtx, client, mkdirScript); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("prepare remote dirs: %w", err)
+	}
+
+	// Upload to a sibling ".new" path to avoid ETXTBSY when the old
+	// binary is still running. The start script below performs an atomic
+	// rename into place after stopping the previous process.
+	remoteBinStaging := remoteBin + ".new"
+	if err := uploadFile(sshCtx, client, localBin, remoteBinStaging, 0o755); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("upload agent binary: %w", err)
+	}
+
+	token := cluster.Agent.Token
+	if token == "" || opts.RotateToken {
+		token = randomHex(24)
+	}
+	requestSecret := strings.TrimSpace(cluster.Agent.RequestSecret)
+	if requestSecret == "" || opts.RotateToken {
+		requestSecret = randomHex(32)
+	}
+
+	certPEM, keyPEM, certFingerprint, err := generateAgentTLSMaterial()
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("generate agent TLS cert: %w", err)
+	}
+	remoteCert := path.Join(remoteBase, "run", "agent-cert.pem")
+	remoteKey := path.Join(remoteBase, "run", "agent-key.pem")
+	if err := uploadBytes(sshCtx, client, certPEM, remoteCert, 0o600); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("upload agent tls cert: %w", err)
+	}
+	if err := uploadBytes(sshCtx, client, keyPEM, remoteKey, 0o600); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("upload agent tls key: %w", err)
+	}
+
+	cfgPayload, err := json.MarshalIndent(map[string]any{
+		"listen_addr":    listenAddress,
+		"token":          token,
+		"request_secret": requestSecret,
+		"tls_enabled":    true,
+		"tls_cert_path":  remoteCert,
+		"tls_key_path":   remoteKey,
+	}, "", "  ")
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("encode remote agent config: %w", err)
+	}
+	cfgPayload = append(cfgPayload, '\n')
+
+	if err := uploadBytes(sshCtx, client, cfgPayload, remoteCfg, 0o600); err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("upload agent config: %w", err)
+	}
+
+	// Stop the previous agent (if any), give it a brief moment to release
+	// the text segment, then atomically rename the staged binary into place
+	// and start the new process. `mv` on the same filesystem is atomic and
+	// safe even if the old inode is still held by the running process —
+	// the path is repointed to the new inode and the old one is unlinked
+	// once the process exits.
+	startScript := fmt.Sprintf(`if [ -f %s ]; then
+  OLDPID="$(cat %s)"
+  if [ -n "$OLDPID" ]; then
+    kill "$OLDPID" >/dev/null 2>&1 || true
+    for i in 1 2 3 4 5 6 7 8 9 10; do
+      kill -0 "$OLDPID" >/dev/null 2>&1 || break
+      sleep 0.2
+    done
+    kill -9 "$OLDPID" >/dev/null 2>&1 || true
+  fi
+fi
+# If any process is still listening on the target agent port (for example
+# from a previous/legacy install path), stop it so the new agent can bind.
+if command -v ss >/dev/null 2>&1; then
+  for P in $(ss -lntp 2>/dev/null | grep -E "[:.]%d[[:space:]]" | sed -n 's/.*pid=\([0-9][0-9]*\).*/\1/p' | sort -u); do
+    [ -n "$P" ] || continue
+    kill "$P" >/dev/null 2>&1 || true
+    for i in 1 2 3 4 5; do
+      kill -0 "$P" >/dev/null 2>&1 || break
+      sleep 0.2
+    done
+    kill -9 "$P" >/dev/null 2>&1 || true
+  done
+fi
+mv -f %s %s
+chmod 0755 %s
+nohup %s --config %s > %s 2>&1 &
+echo $! > %s
+cat %s`,
+		shellQuote(remotePID), shellQuote(remotePID),
+		agentPort,
+		shellQuote(remoteBinStaging), shellQuote(remoteBin),
+		shellQuote(remoteBin),
+		shellQuote(remoteBin), shellQuote(remoteCfg), shellQuote(remoteLog),
+		shellQuote(remotePID), shellQuote(remotePID))
+	pidOut, err := runRemoteCommand(sshCtx, client, startScript)
+	if err != nil {
+		return Cluster{}, BootstrapResult{}, fmt.Errorf("start remote agent: %w", err)
+	}
+	pid := strings.TrimSpace(pidOut)
+
+	cluster.Agent = AgentInstall{
+		Installed:       true,
+		Version:         expectedAgentVersion(),
+		RemoteBinary:    remoteBin,
+		RemoteConfig:    remoteCfg,
+		RemoteLog:       remoteLog,
+		RemotePIDFile:   remotePID,
+		ListenAddress:   listenAddress,
+		Port:            agentPort,
+		Token:           token,
+		RequestSecret:   requestSecret,
+		TLSEnabled:      true,
+		TLSCertPath:     remoteCert,
+		TLSKeyPath:      remoteKey,
+		TLSFingerprint:  certFingerprint,
+		LastBootstrapAt: s.now().UTC(),
+	}
+	cluster.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = cluster
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, BootstrapResult{}, err
+	}
+
+	s.CloseTunnelClient(cluster.ID)
+	ping := s.probeAgent(ctx, cluster)
+	if strings.TrimSpace(ping.Version) != "" && ping.Version != cluster.Agent.Version {
+		cluster.Agent.Version = ping.Version
+		cluster.UpdatedAt = s.now().UTC()
+		reg.Clusters[idx] = cluster
+		_ = s.store.Save(reg)
+	}
+	if !ping.Reachable && !opts.AllowAgentProbe {
+		return Cluster{}, BootstrapResult{RemoteOS: remoteOS, RemoteArch: remoteArch, PID: pid, AgentPing: ping}, fmt.Errorf("agent deployed but probe failed: %s", ping.Error)
+	}
+
+	result := BootstrapResult{
+		RemoteOS:   remoteOS,
+		RemoteArch: remoteArch,
+		PID:        pid,
+		AgentPing:  ping,
+	}
+	return cluster, result, nil
+}
+
+func (s *Service) PingAgent(ctx context.Context, selector string) (AgentPingResult, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return AgentPingResult{}, err
+	}
+	if !cluster.Agent.Installed {
+		return AgentPingResult{}, errors.New("agent is not installed on this cluster")
+	}
+	return s.probeAgent(ctx, cluster), nil
+}
+
+func (s *Service) AdoptAgentAuth(ctx context.Context, selector string) (Cluster, AgentPingResult, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, AgentPingResult{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, AgentPingResult{}, err
+	}
+
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return Cluster{}, AgentPingResult{}, err
+	}
+	defer client.Close()
+
+	remoteCfg := strings.TrimSpace(c.Agent.RemoteConfig)
+	if remoteCfg == "" {
+		home, homeErr := remoteHomeDir(ctx, client)
+		if homeErr != nil {
+			return Cluster{}, AgentPingResult{}, fmt.Errorf("resolve remote home: %w", homeErr)
+		}
+		remoteCfg = path.Join(home, ".pxmon", "run", "agent.json")
+	}
+	cfgText, err := runRemoteCommand(ctx, client, "cat "+shellQuote(remoteCfg))
+	if err != nil {
+		return Cluster{}, AgentPingResult{}, fmt.Errorf("read remote agent config: %w", err)
+	}
+
+	var remote struct {
+		ListenAddr    string `json:"listen_addr"`
+		Token         string `json:"token"`
+		RequestSecret string `json:"request_secret"`
+		TLSEnabled    bool   `json:"tls_enabled"`
+		TLSCertPath   string `json:"tls_cert_path"`
+		TLSKeyPath    string `json:"tls_key_path"`
+	}
+	if err := json.Unmarshal([]byte(cfgText), &remote); err != nil {
+		return Cluster{}, AgentPingResult{}, fmt.Errorf("parse remote agent config: %w", err)
+	}
+	if strings.TrimSpace(remote.Token) == "" {
+		return Cluster{}, AgentPingResult{}, errors.New("remote agent config has empty token")
+	}
+
+	fingerprint := c.Agent.TLSFingerprint
+	if remote.TLSEnabled {
+		certPath := strings.TrimSpace(remote.TLSCertPath)
+		if certPath == "" {
+			return Cluster{}, AgentPingResult{}, errors.New("remote agent TLS enabled but cert path is empty")
+		}
+		certPEM, certErr := runRemoteCommand(ctx, client, "cat "+shellQuote(certPath))
+		if certErr != nil {
+			return Cluster{}, AgentPingResult{}, fmt.Errorf("read remote agent TLS cert: %w", certErr)
+		}
+		fp, fpErr := fingerprintCertPEM([]byte(certPEM))
+		if fpErr != nil {
+			return Cluster{}, AgentPingResult{}, fpErr
+		}
+		fingerprint = fp
+	}
+
+	port := c.Agent.Port
+	if p, pErr := parsePortFromListen(remote.ListenAddr); pErr == nil {
+		port = p
+	}
+	if port == 0 {
+		port = defaultAgentPort
+	}
+
+	c.Agent.Installed = true
+	c.Agent.Version = expectedAgentVersion()
+	c.Agent.RemoteConfig = remoteCfg
+	c.Agent.ListenAddress = remote.ListenAddr
+	c.Agent.Port = port
+	c.Agent.Token = strings.TrimSpace(remote.Token)
+	c.Agent.RequestSecret = strings.TrimSpace(remote.RequestSecret)
+	c.Agent.TLSEnabled = remote.TLSEnabled
+	c.Agent.TLSCertPath = strings.TrimSpace(remote.TLSCertPath)
+	c.Agent.TLSKeyPath = strings.TrimSpace(remote.TLSKeyPath)
+	c.Agent.TLSFingerprint = fingerprint
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, AgentPingResult{}, err
+	}
+
+	s.CloseTunnelClient(c.ID)
+	ping := s.probeAgent(ctx, c)
+	if strings.TrimSpace(ping.Version) != "" {
+		c.Agent.Version = ping.Version
+		c.UpdatedAt = s.now().UTC()
+		reg.Clusters[idx] = c
+		_ = s.store.Save(reg)
+	}
+	return c, ping, nil
+}
+
+func (s *Service) AgentStats(ctx context.Context, selector string) (map[string]any, error) {
+	body, err := s.agentStatsBody(ctx, selector)
+	if err != nil {
+		return nil, err
+	}
+
+	var payload map[string]any
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return nil, err
+	}
+	return payload, nil
+}
+
+func (s *Service) AgentStatsTyped(ctx context.Context, selector string) (agent.StatsResponse, error) {
+	body, err := s.agentStatsBody(ctx, selector)
+	if err != nil {
+		return agent.StatsResponse{}, err
+	}
+
+	var payload agent.StatsResponse
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return agent.StatsResponse{}, err
+	}
+	return payload, nil
+}
+
+func (s *Service) agentStatsBody(ctx context.Context, selector string) ([]byte, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return nil, err
+	}
+	if !cluster.Agent.Installed {
+		return nil, errors.New("agent is not installed on this cluster")
+	}
+
+	ac, err := s.newAgentClient(ctx, cluster, 8*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	defer ac.Close()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, ac.target+"/api/v1/stats", nil)
+	if err != nil {
+		return nil, err
+	}
+	applyAgentRequestAuth(req, cluster)
+
+	resp, err := ac.http.Do(req)
+	if err != nil {
+		if normalizeTransport(cluster.Transport) == TransportIPFabric {
+			s.CloseTunnelClient(cluster.ID)
+		}
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 300 {
+		msg := strings.TrimSpace(string(body))
+		if len(msg) > 200 {
+			msg = msg[:200]
+		}
+		return nil, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
+	}
+	return body, nil
+}
+
+func (s *Service) probeAgent(ctx context.Context, c Cluster) AgentPingResult {
+	result := AgentPingResult{
+		Reachable: false,
+		CheckedAt: s.now().UTC(),
+	}
+
+	if c.Agent.Port == 0 {
+		result.Endpoint = fmt.Sprintf("%s://%s:%d/api/v1/ping", agentScheme(c), c.Host, c.Agent.Port)
+		result.Error = "agent port is not set"
+		return result
+	}
+
+	ac, err := s.newAgentClient(ctx, c, 6*time.Second)
+	if err != nil {
+		result.Endpoint = fmt.Sprintf("%s://%s:%d/api/v1/ping", agentScheme(c), c.Host, c.Agent.Port)
+		result.Error = err.Error()
+		return result
+	}
+	defer ac.Close()
+
+	result.Endpoint = ac.target + "/api/v1/ping"
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, result.Endpoint, nil)
+	if err != nil {
+		result.Error = err.Error()
+		return result
+	}
+	applyAgentRequestAuth(req, c)
+
+	resp, err := ac.http.Do(req)
+	if err != nil {
+		if normalizeTransport(c.Transport) == TransportIPFabric {
+			s.CloseTunnelClient(c.ID)
+		}
+		result.Error = err.Error()
+		return result
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(resp.Body)
+
+	result.StatusCode = resp.StatusCode
+	result.Reachable = true
+	if len(body) > 0 {
+		var payload struct {
+			Version string `json:"version"`
+		}
+		if err := json.Unmarshal(body, &payload); err == nil {
+			result.Version = strings.TrimSpace(payload.Version)
+		}
+	}
+	if resp.StatusCode >= 400 {
+		result.Error = fmt.Sprintf("HTTP %d", resp.StatusCode)
+	}
+
+	return result
+}
+
+func (s *Service) ExpectedAgentVersion() string {
+	return expectedAgentVersion()
+}
+
+func expectedAgentVersion() string {
+	v := strings.TrimSpace(agent.Version)
+	if v == "" {
+		return "dev"
+	}
+	return v
+}
+
+func (s *Service) dialSSH(ctx context.Context, c Cluster, passwordOverride, keyPassphraseOverride string) (*ssh.Client, error) {
+	client, err := s.dialSSHOnce(ctx, c, passwordOverride, keyPassphraseOverride)
+	if err == nil {
+		return client, nil
+	}
+
+	// Auto-heal known_hosts problems by fetching and appending the current server key,
+	// then retrying once.
+	if c.InsecureHostKey || !isKnownHostsError(err) {
+		return nil, err
+	}
+	if fixErr := s.ensureKnownHostEntry(ctx, c); fixErr != nil {
+		return nil, fmt.Errorf("%w (auto host key update failed: %v)", err, fixErr)
+	}
+
+	return s.dialSSHOnce(ctx, c, passwordOverride, keyPassphraseOverride)
+}
+
+func (s *Service) dialSSHOnce(ctx context.Context, c Cluster, passwordOverride, keyPassphraseOverride string) (*ssh.Client, error) {
+	password := strings.TrimSpace(passwordOverride)
+	if password == "" {
+		password = c.Password
+	}
+
+	keyPassphrase := strings.TrimSpace(keyPassphraseOverride)
+	if keyPassphrase == "" {
+		keyPassphrase = c.KeyPassphrase
+	}
+	if keyPassphrase == "" && strings.TrimSpace(c.KeyPassphraseFile) != "" {
+		filePassphrase, err := readStoredKeyPassphraseFile(c.KeyPassphraseFile)
+		if err != nil {
+			return nil, err
+		}
+		keyPassphrase = filePassphrase
+	}
+
+	hostKeyCallback, err := s.hostKeyCallback(c.InsecureHostKey)
+	if err != nil {
+		return nil, err
+	}
+
+	auth, err := sshAuthMethods(c, password, keyPassphrase)
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &ssh.ClientConfig{
+		User:            c.User,
+		Auth:            auth,
+		HostKeyCallback: hostKeyCallback,
+		Timeout:         defaultProbeTO,
+	}
+
+	addr := net.JoinHostPort(c.Host, strconv.Itoa(c.Port))
+	dialer := &net.Dialer{Timeout: defaultProbeTO}
+	conn, err := dialer.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+
+	sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, cfg)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return ssh.NewClient(sshConn, chans, reqs), nil
+}
+
+func (s *Service) hostKeyCallback(insecure bool) (ssh.HostKeyCallback, error) {
+	if insecure {
+		return ssh.InsecureIgnoreHostKey(), nil //nolint:gosec
+	}
+
+	knownHostsPath, err := knownHostsPath()
+	if err != nil {
+		return nil, err
+	}
+	if err := ensureKnownHostsFile(knownHostsPath); err != nil {
+		return nil, err
+	}
+
+	cb, err := knownhosts.New(knownHostsPath)
+	if err != nil {
+		return nil, fmt.Errorf("parse known_hosts: %w", err)
+	}
+	return cb, nil
+}
+
+func knownHostsPath() (string, error) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", fmt.Errorf("resolve home directory: %w", err)
+	}
+	return filepath.Join(home, ".ssh", "known_hosts"), nil
+}
+
+func ensureKnownHostsFile(path string) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return fmt.Errorf("create ~/.ssh directory: %w", err)
+	}
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
+	if err != nil {
+		return fmt.Errorf("create known_hosts: %w", err)
+	}
+	return f.Close()
+}
+
+func (s *Service) ensureKnownHostEntry(ctx context.Context, c Cluster) error {
+	path, err := knownHostsPath()
+	if err != nil {
+		return err
+	}
+	if err := ensureKnownHostsFile(path); err != nil {
+		return err
+	}
+
+	key, err := fetchHostKey(ctx, c.Host, c.Port)
+	if err != nil {
+		return err
+	}
+
+	hostEntry := knownHostAddress(c.Host, c.Port)
+	line := knownhosts.Line([]string{hostEntry}, key)
+	existing, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("read known_hosts: %w", err)
+	}
+	if bytes.Contains(existing, []byte(line)) {
+		return nil
+	}
+
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600)
+	if err != nil {
+		return fmt.Errorf("open known_hosts for update: %w", err)
+	}
+	defer f.Close()
+
+	if len(existing) > 0 && existing[len(existing)-1] != '\n' {
+		if _, err := f.Write([]byte("\n")); err != nil {
+			return fmt.Errorf("write known_hosts newline: %w", err)
+		}
+	}
+	if _, err := f.WriteString(line + "\n"); err != nil {
+		return fmt.Errorf("append host key to known_hosts: %w", err)
+	}
+	return nil
+}
+
+func fetchHostKey(ctx context.Context, host string, port int) (ssh.PublicKey, error) {
+	addr := net.JoinHostPort(host, strconv.Itoa(port))
+	dialer := &net.Dialer{Timeout: defaultProbeTO}
+	conn, err := dialer.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("dial host for key scan: %w", err)
+	}
+	defer conn.Close()
+
+	var serverKey ssh.PublicKey
+	cfg := &ssh.ClientConfig{
+		User: "pxmon-keyscan",
+		Auth: []ssh.AuthMethod{ssh.Password("invalid-password")},
+		HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
+			serverKey = key
+			return nil
+		},
+		Timeout: defaultProbeTO,
+	}
+
+	sshConn, _, _, err := ssh.NewClientConn(conn, addr, cfg)
+	if sshConn != nil {
+		_ = sshConn.Close()
+	}
+	if serverKey == nil {
+		if err != nil {
+			return nil, fmt.Errorf("fetch host key: %w", err)
+		}
+		return nil, errors.New("fetch host key: empty server key")
+	}
+	return serverKey, nil
+}
+
+func knownHostAddress(host string, port int) string {
+	if port == 22 {
+		return host
+	}
+	return fmt.Sprintf("[%s]:%d", host, port)
+}
+
+func isKnownHostsError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var keyErr *knownhosts.KeyError
+	if errors.As(err, &keyErr) {
+		return true
+	}
+	lower := strings.ToLower(err.Error())
+	return strings.Contains(lower, "knownhosts:")
+}
+
+func remoteHomeDir(ctx context.Context, client *ssh.Client) (string, error) {
+	out, err := runRemoteCommand(ctx, client, `printf %s "$HOME"`)
+	if err != nil {
+		return "", err
+	}
+	home := strings.TrimSpace(out)
+	if home == "" || !strings.HasPrefix(home, "/") {
+		return "", fmt.Errorf("unexpected HOME value: %q", home)
+	}
+	return home, nil
+}
+
+func sshAuthMethods(c Cluster, password, keyPassphrase string) ([]ssh.AuthMethod, error) {
+	switch c.AuthMethod {
+	case AuthMethodPassword:
+		if password == "" {
+			return nil, errors.New("password auth selected but password is empty; provide --password")
+		}
+		ki := ssh.KeyboardInteractive(func(user, instruction string, questions []string, echos []bool) ([]string, error) {
+			answers := make([]string, len(questions))
+			for i := range questions {
+				answers[i] = password
+			}
+			return answers, nil
+		})
+		return []ssh.AuthMethod{ssh.Password(password), ki}, nil
+	case AuthMethodKey:
+		if strings.TrimSpace(c.KeyPath) == "" {
+			return nil, errors.New("key auth selected but key path is empty")
+		}
+		keyPath, err := expandPath(c.KeyPath)
+		if err != nil {
+			return nil, err
+		}
+		pemBytes, err := os.ReadFile(keyPath)
+		if err != nil {
+			return nil, fmt.Errorf("read SSH key: %w", err)
+		}
+
+		var signer ssh.Signer
+		if keyPassphrase != "" {
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(pemBytes, []byte(keyPassphrase))
+		} else {
+			signer, err = ssh.ParsePrivateKey(pemBytes)
+		}
+		if err != nil {
+			if strings.Contains(strings.ToLower(err.Error()), "encrypted") {
+				return nil, errors.New("encrypted SSH key requires --key-passphrase")
+			}
+			return nil, fmt.Errorf("parse SSH key: %w", err)
+		}
+		return []ssh.AuthMethod{ssh.PublicKeys(signer)}, nil
+	default:
+		return nil, fmt.Errorf("unsupported auth method %q", c.AuthMethod)
+	}
+}
+
+func readStoredKeyPassphraseFile(path string) (string, error) {
+	expanded, err := expandPath(path)
+	if err != nil {
+		return "", err
+	}
+	data, err := os.ReadFile(expanded)
+	if err != nil {
+		return "", fmt.Errorf("read key passphrase file: %w", err)
+	}
+	passphrase := strings.TrimRight(string(data), "\r\n")
+	if passphrase == "" {
+		return "", errors.New("key passphrase file is empty")
+	}
+	return passphrase, nil
+}
+
+func fingerprintCertPEM(certPEM []byte) (string, error) {
+	block, _ := pem.Decode(certPEM)
+	if block == nil {
+		return "", errors.New("decode remote agent TLS cert: no PEM block found")
+	}
+	if block.Type != "CERTIFICATE" {
+		return "", fmt.Errorf("decode remote agent TLS cert: unexpected PEM type %q", block.Type)
+	}
+	sum := sha256.Sum256(block.Bytes)
+	return hex.EncodeToString(sum[:]), nil
+}
+
+func runRemoteCommand(ctx context.Context, client *ssh.Client, script string) (string, error) {
+	session, err := client.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer session.Close()
+
+	type output struct {
+		buf []byte
+		err error
+	}
+
+	ch := make(chan output, 1)
+	go func() {
+		cmd := "sh -lc " + shellQuote(script)
+		buf, runErr := session.CombinedOutput(cmd)
+		ch <- output{buf: buf, err: runErr}
+	}()
+
+	select {
+	case <-ctx.Done():
+		return "", ctx.Err()
+	case out := <-ch:
+		if out.err != nil {
+			message := strings.TrimSpace(string(out.buf))
+			if message != "" {
+				return "", fmt.Errorf("%w: %s", out.err, message)
+			}
+			return "", out.err
+		}
+		return string(out.buf), nil
+	}
+}
+
+func uploadFile(ctx context.Context, client *ssh.Client, localPath, remotePath string, mode os.FileMode) error {
+	f, err := os.Open(localPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return uploadReader(ctx, client, f, remotePath, mode)
+}
+
+func uploadBytes(ctx context.Context, client *ssh.Client, data []byte, remotePath string, mode os.FileMode) error {
+	return uploadReader(ctx, client, bytes.NewReader(data), remotePath, mode)
+}
+
+func uploadReader(ctx context.Context, client *ssh.Client, r io.Reader, remotePath string, mode os.FileMode) error {
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	var stderr bytes.Buffer
+	session.Stderr = &stderr
+
+	stdin, err := session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	script := fmt.Sprintf("cat > %s && chmod %o %s", shellQuote(remotePath), mode.Perm(), shellQuote(remotePath))
+	if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
+		return err
+	}
+
+	copyDone := make(chan error, 1)
+	go func() {
+		_, copyErr := io.Copy(stdin, r)
+		if closeErr := stdin.Close(); closeErr != nil && copyErr == nil {
+			copyErr = closeErr
+		}
+		copyDone <- copyErr
+	}()
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-copyDone:
+		if err != nil {
+			_ = session.Wait()
+			msg := strings.TrimSpace(stderr.String())
+			if msg != "" {
+				return fmt.Errorf("upload stream failed: %w: %s", err, msg)
+			}
+			return fmt.Errorf("upload stream failed: %w", err)
+		}
+	}
+
+	if err := session.Wait(); err != nil {
+		msg := strings.TrimSpace(stderr.String())
+		if msg != "" {
+			return fmt.Errorf("%w: %s", err, msg)
+		}
+		return err
+	}
+
+	return nil
+}
+
+func buildAgentBinary(goos, goarch string) (string, func(), error) {
+	if _, err := os.Stat("./cmd/pxmon-agent"); err != nil {
+		return "", func() {}, errors.New("cmd/pxmon-agent not found; run bootstrap from project root or provide --agent-bin")
+	}
+
+	tmpDir, err := os.MkdirTemp("", "pxmon-agent-build-*")
+	if err != nil {
+		return "", func() {}, err
+	}
+
+	bin := filepath.Join(tmpDir, "pxmon-agent")
+	cmd := exec.Command("go", "build", "-o", bin, "./cmd/pxmon-agent")
+	cmd.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH="+goarch)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return "", func() {}, fmt.Errorf("build agent failed: %w: %s", err, strings.TrimSpace(string(out)))
+	}
+
+	cleanup := func() { _ = os.RemoveAll(tmpDir) }
+	return bin, cleanup, nil
+}
+
+func mapRuntime(unameOS, unameArch string) (string, string, error) {
+	osName := strings.ToLower(strings.TrimSpace(unameOS))
+	archName := strings.ToLower(strings.TrimSpace(unameArch))
+
+	var goos string
+	switch osName {
+	case "linux":
+		goos = "linux"
+	case "darwin":
+		goos = "darwin"
+	default:
+		return "", "", fmt.Errorf("unsupported remote OS %q", unameOS)
+	}
+
+	var goarch string
+	switch archName {
+	case "x86_64", "amd64":
+		goarch = "amd64"
+	case "i386", "i686", "386":
+		goarch = "386"
+	case "aarch64", "arm64":
+		goarch = "arm64"
+	case "armv7l", "armv6l", "arm":
+		goarch = "arm"
+	default:
+		return "", "", fmt.Errorf("unsupported remote architecture %q", unameArch)
+	}
+
+	return goos, goarch, nil
+}
+
+func parsePortFromListen(listenAddr string) (int, error) {
+	addr := strings.TrimSpace(listenAddr)
+	if addr == "" {
+		return defaultAgentPort, nil
+	}
+
+	host, portStr, err := net.SplitHostPort(addr)
+	if err != nil {
+		if strings.Count(addr, ":") == 1 {
+			parts := strings.Split(addr, ":")
+			host = parts[0]
+			portStr = parts[1]
+		} else {
+			return 0, fmt.Errorf("invalid listen address %q, expected host:port", addr)
+		}
+	}
+
+	_ = host
+	p, err := strconv.Atoi(portStr)
+	if err != nil || p < 1 || p > 65535 {
+		return 0, fmt.Errorf("invalid listen port in %q", addr)
+	}
+	return p, nil
+}
+
+func findCluster(reg Registry, selector string) (Cluster, int, error) {
+	sel := strings.TrimSpace(selector)
+	if sel == "" {
+		sel = reg.ActiveClusterID
+	}
+	if sel == "" {
+		return Cluster{}, -1, ErrNoActiveCluster
+	}
+
+	for i, c := range reg.Clusters {
+		if c.ID == sel || strings.EqualFold(c.Name, sel) {
+			return c, i, nil
+		}
+	}
+
+	return Cluster{}, -1, ErrClusterNotFound
+}
+
+func existsTarget(clusters []Cluster, host string, port int, user string) bool {
+	for _, c := range clusters {
+		if strings.EqualFold(c.Host, host) && c.Port == port && strings.EqualFold(c.User, user) {
+			return true
+		}
+	}
+	return false
+}
+
+func newClusterID() string {
+	b := make([]byte, 8)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Sprintf("clu_%d", time.Now().UnixNano())
+	}
+	return "clu_" + hex.EncodeToString(b)
+}
+
+func randomHex(byteLen int) string {
+	b := make([]byte, byteLen)
+	if _, err := rand.Read(b); err != nil {
+		return fmt.Sprintf("tok_%d", time.Now().UnixNano())
+	}
+	return hex.EncodeToString(b)
+}
+
+func generateAgentTLSMaterial() (certPEM []byte, keyPEM []byte, fingerprint string, err error) {
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 62))
+	if err != nil {
+		return nil, nil, "", err
+	}
+	tpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName: "pxmon-agent",
+		},
+		NotBefore:             time.Now().UTC().Add(-10 * time.Minute),
+		NotAfter:              time.Now().UTC().Add(3650 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		DNSNames:              []string{"pxmon-agent", "localhost"},
+		IPAddresses:           []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, pub, priv)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	keyRaw, err := x509.MarshalPKCS8PrivateKey(priv)
+	if err != nil {
+		return nil, nil, "", err
+	}
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw})
+	sum := sha256.Sum256(der)
+	return certPEM, keyPEM, hex.EncodeToString(sum[:]), nil
+}
+
+func expandPath(path string) (string, error) {
+	p := strings.TrimSpace(path)
+	if p == "" {
+		return "", errors.New("empty path")
+	}
+	if strings.HasPrefix(p, "~") {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return "", err
+		}
+		if p == "~" {
+			p = home
+		} else {
+			p = filepath.Join(home, strings.TrimPrefix(p, "~/"))
+		}
+	}
+	return p, nil
+}
+
+func splitNonEmptyLines(s string) []string {
+	parts := strings.Split(s, "\n")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		p = strings.TrimSpace(p)
+		if p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func softwareProbeScript() string {
+	return strings.Join([]string{
+		"probe() { if command -v \"$1\" >/dev/null 2>&1; then echo 1; else echo 0; fi; }",
+		"echo bird=$(probe birdc)",
+		"echo frr=$(probe vtysh)",
+		"if command -v virsh >/dev/null 2>&1 || command -v qemu-system-x86_64 >/dev/null 2>&1; then echo kvm=1; else echo kvm=0; fi",
+		"echo lxc=$(probe lxc)",
+		"echo lxd=$(probe lxd)",
+		"if command -v birdc >/dev/null 2>&1; then echo ver_bird=$(birdc --version 2>/dev/null | head -n1); fi",
+		"if command -v vtysh >/dev/null 2>&1; then echo ver_frr=$(vtysh -v 2>/dev/null | head -n1); fi",
+		"if command -v virsh >/dev/null 2>&1; then echo ver_kvm=$(virsh --version 2>/dev/null | head -n1); fi",
+		"if command -v lxc >/dev/null 2>&1; then echo ver_lxc=$(lxc --version 2>/dev/null | head -n1); fi",
+		"if command -v lxd >/dev/null 2>&1; then echo ver_lxd=$(lxd --version 2>/dev/null | head -n1); fi",
+	}, "; ")
+}
+
+func parseSoftwareProbe(out string, now time.Time) SoftwareInfo {
+	info := SoftwareInfo{
+		DetectedAt: now,
+		Versions:   map[string]string{},
+	}
+	for _, line := range splitNonEmptyLines(out) {
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		k := strings.ToLower(strings.TrimSpace(parts[0]))
+		v := strings.TrimSpace(parts[1])
+		switch k {
+		case "bird":
+			info.Bird = v == "1" || strings.EqualFold(v, "true")
+		case "frr":
+			info.FRR = v == "1" || strings.EqualFold(v, "true")
+		case "kvm":
+			info.KVM = v == "1" || strings.EqualFold(v, "true")
+		case "lxc":
+			info.LXC = v == "1" || strings.EqualFold(v, "true")
+		case "lxd":
+			info.LXD = v == "1" || strings.EqualFold(v, "true")
+		case "ver_bird":
+			if v != "" {
+				info.Versions["bird"] = v
+			}
+		case "ver_frr":
+			if v != "" {
+				info.Versions["frr"] = v
+			}
+		case "ver_kvm":
+			if v != "" {
+				info.Versions["kvm"] = v
+			}
+		case "ver_lxc":
+			if v != "" {
+				info.Versions["lxc"] = v
+			}
+		case "ver_lxd":
+			if v != "" {
+				info.Versions["lxd"] = v
+			}
+		}
+	}
+	if len(info.Versions) == 0 {
+		info.Versions = nil
+	}
+	return info
+}
+
+func softwareProbeKnown(info SoftwareInfo) bool {
+	if !info.DetectedAt.IsZero() {
+		return true
+	}
+	if info.Bird || info.FRR || info.KVM || info.LXC || info.LXD {
+		return true
+	}
+	return len(info.Versions) > 0
+}
+
+func isPluginToolSupported(info SoftwareInfo, tool string) bool {
+	switch strings.ToLower(strings.TrimSpace(tool)) {
+	case "bird":
+		return info.Bird
+	case "frr":
+		return info.FRR
+	case "kvm":
+		return info.KVM
+	case "lxc":
+		return info.LXC || info.LXD
+	case "lxd":
+		return info.LXD || info.LXC
+	default:
+		return false
+	}
+}
+
+func pluginScript(tool, action string, args []string) (string, error) {
+	tool = strings.ToLower(strings.TrimSpace(tool))
+	action = strings.ToLower(strings.TrimSpace(action))
+
+	switch tool {
+	case "kvm":
+		return kvmPluginScript(action, args)
+	case "lxc", "lxd":
+		return lxcPluginScript(action, args)
+	case "bird":
+		return birdPluginScript(action)
+	case "frr":
+		return frrPluginScript(action)
+	default:
+		return "", fmt.Errorf("unsupported plugin tool %q", tool)
+	}
+}
+
+func kvmPluginScript(action string, args []string) (string, error) {
+	mustDomain := func() (string, error) {
+		if len(args) < 1 || strings.TrimSpace(args[0]) == "" {
+			return "", errors.New("domain is required")
+		}
+		return shellQuote(strings.TrimSpace(args[0])), nil
+	}
+
+	switch action {
+	case "", "list", "domains":
+		return "virsh list --all", nil
+	case "start":
+		d, err := mustDomain()
+		if err != nil {
+			return "", err
+		}
+		return "virsh start " + d, nil
+	case "stop", "shutdown":
+		d, err := mustDomain()
+		if err != nil {
+			return "", err
+		}
+		return "virsh shutdown " + d, nil
+	case "reboot", "restart":
+		d, err := mustDomain()
+		if err != nil {
+			return "", err
+		}
+		return "virsh reboot " + d, nil
+	case "destroy", "force-stop":
+		d, err := mustDomain()
+		if err != nil {
+			return "", err
+		}
+		return "virsh destroy " + d, nil
+	case "top":
+		return strings.Join([]string{
+			`names="$(mktemp)"; states="$(mktemp)"; raw="$(mktemp)"; trap 'rm -f "$names" "$states" "$raw"' EXIT`,
+			`virsh list --all --name 2>/dev/null | awk 'NF>0{print $1}' > "$names"`,
+			`virsh list --all 2>/dev/null | awk 'NR>2 && NF>=3 {name=$2; state=$3; for(i=4;i<=NF;i++) state=state" "$i; print name "\t" state}' > "$states"`,
+			`while IFS= read -r d; do [ -z "$d" ] && continue; state=$(awk -F'\t' -v n="$d" '$1==n{print $2; found=1; exit} END{if(!found) print "-"}' "$states"); info=$(virsh dominfo "$d" 2>/dev/null || true); vcpu=$(printf "%s\n" "$info" | awk -F: '/^CPU\(s\):/{gsub(/[[:space:]]+/,"",$2); print $2; exit}'); ram_kib=$(printf "%s\n" "$info" | awk -F: '/^Max memory:/{gsub(/[^0-9]/,"",$2); print $2; exit}'); [ -z "$vcpu" ] && vcpu=-1; [ -z "$ram_kib" ] && ram_kib=-1; disk_cap=0; disk_alloc=0; has_disk=0; dl="$(mktemp)"; virsh domblklist "$d" --details 2>/dev/null | awk 'NR>2 && $2=="disk" && $3!=""{t=$3; s=$4; if(s=="") s="-"; print t "\t" s}' > "$dl"; tab="$(printf '\t')"; while IFS="$tab" read -r dev src; do [ -z "$dev" ] && continue; binfo=$(virsh domblkinfo "$d" "$dev" 2>/dev/null || true); cap=$(printf "%s\n" "$binfo" | awk '/^Capacity:/{print $2; exit}'); alloc=$(printf "%s\n" "$binfo" | awk '/^Allocation:/{print $2; exit}'); if [ -z "$cap" ] && [ -n "$src" ] && [ "$src" != "-" ]; then if command -v qemu-img >/dev/null 2>&1; then cap=$(qemu-img info --output=json "$src" 2>/dev/null | awk -F: '/"virtual-size"[[:space:]]*:/{gsub(/[^0-9]/,"",$2); print $2; exit}'); if [ -z "$cap" ]; then cap=$(qemu-img info "$src" 2>/dev/null | awk -F'[()]' '/virtual size:/{gsub(/[^0-9]/,"",$2); print $2; exit}'); fi; fi; fi; if [ -z "$alloc" ] && [ -n "$src" ] && [ "$src" != "-" ] && [ -f "$src" ]; then alloc=$(wc -c < "$src" 2>/dev/null | tr -d '[:space:]'); fi; if [ -n "$cap" ] || [ -n "$alloc" ]; then has_disk=1; [ -z "$cap" ] && cap=0; [ -z "$alloc" ] && alloc=0; disk_cap=$((disk_cap + cap)); disk_alloc=$((disk_alloc + alloc)); fi; done < "$dl"; rm -f "$dl"; if [ "$has_disk" -eq 0 ]; then disk_cap=-1; disk_alloc=-1; fi; printf "%s\t%s\t%s\t%s\t%s\t%s\n" "$d" "$state" "$vcpu" "$ram_kib" "$disk_cap" "$disk_alloc" >> "$raw"; done < "$names"`,
+			`sort -t "$(printf '\t')" -k1,1 "$raw" | awk -F'\t' 'BEGIN{printf "%-34s %-12s %7s %11s %11s %11s\n","DOMAIN","STATE","VCPU","RAM_MAX","DISK_CAP","DISK_ALLOC"} {dom=fit($1,34); st=fit($2,12); vc=numOrDash($3); ram=humanKiBOrDash($4); dcap=humanBytesOrDash($5); dalloc=humanBytesOrDash($6); printf "%-34s %-12s %7s %11s %11s %11s\n",dom,st,vc,ram,dcap,dalloc} function numOrDash(v){if(v==""||v<0)return "-"; return sprintf("%d", v+0)} function humanKiBOrDash(v){if(v==""||v<0)return "-"; return human((v+0)*1024)} function humanBytesOrDash(v){if(v==""||v<0)return "-"; return human(v+0)} function human(n, u){if(n==0)return "0B"; split("B KiB MiB GiB TiB PiB",u," "); i=1; while(n>=1024 && i<6){n/=1024; i++} if(n>=10 || i==1) return sprintf("%.0f%s",n,u[i]); return sprintf("%.1f%s",n,u[i]);} function fit(s,w){if(length(s)<=w)return s; return substr(s,1,w-1)"…"}'`,
+		}, "; "), nil
+	case "net-top", "net":
+		return strings.Join([]string{
+			`names="$(mktemp)"; states="$(mktemp)"; prev="$(mktemp)"; curr="$(mktemp)"; samples="$(mktemp)"; merged="$(mktemp)"; trap 'rm -f "$names" "$states" "$prev" "$curr" "$samples" "$merged"' EXIT`,
+			`virsh list --all --name 2>/dev/null | awk 'NF>0{print $1}' > "$names"`,
+			`virsh list --all 2>/dev/null | awk 'NR>2 && NF>=3 {name=$2; state=$3; for(i=4;i<=NF;i++) state=state" "$i; print name "\t" state}' > "$states"`,
+			`collect_net_snapshot() { out="$1"; virsh domstats --raw --interface 2>/dev/null | awk 'BEGIN{OFS="\t"; dom=""; q=sprintf("%c",39)} /^Domain:[[:space:]]+/ {dom=$2; if(substr(dom,1,1)==q) dom=substr(dom,2); if(length(dom)>0 && substr(dom,length(dom),1)==q) dom=substr(dom,1,length(dom)-1); seen[dom]=1; next} dom==""{next} /^net\.[0-9]+\.rx\.bytes=/{split($0,a,"="); rx[dom]+=a[2]; hasRx[dom]=1; next} /^net\.[0-9]+\.tx\.bytes=/{split($0,a,"="); tx[dom]+=a[2]; hasTx[dom]=1; next} END{for(d in seen){nr=(hasRx[d]!=""?rx[d]:-1); nt=(hasTx[d]!=""?tx[d]:-1); print d, nr, nt}}' > "$out"; }`,
+			`collect_net_snapshot "$prev"; : > "$samples"; loops=4; intv=0.3; i=1; while [ "$i" -le "$loops" ]; do sleep "$intv"; collect_net_snapshot "$curr"; awk -F'\t' -v intv="$intv" 'NR==FNR{prx[$1]=$2; ptx[$1]=$3; next} {d=$1; rx=$2+0; tx=$3+0; if(!(d in prx)||prx[d]<0||ptx[d]<0||rx<0||tx<0){next} drx=rx-prx[d]; if(drx<0)drx=0; dtx=tx-ptx[d]; if(dtx<0)dtx=0; rxm=drx*8/intv/1000000; txm=dtx*8/intv/1000000; tot=rxm+txm; printf "%s\t%.6f\t%.6f\t%.6f\t%.0f\t%.0f\n", d, tot, rxm, txm, rx, tx}' "$prev" "$curr" >> "$samples"; cp "$curr" "$prev"; i=$((i+1)); done`,
+			`awk -F'\t' 'NR==FNR{st[$1]=$2; next} {d=$1; n[d]++; tot[d,n[d]]=$2+0; rx[d]=$3+0; tx[d]=$4+0; rxTot[d]=$5+0; txTot[d]=$6+0; seen[d]=1} END{while((getline dom < "'"$names"'")>0){if(dom=="")continue; s=st[dom]; if(s=="") s="-"; if(seen[dom]==""){print dom"\t"s"\t-1\t-1\t-1\t-1\t-1\t-1"; continue} k=n[dom]; delete arr; for(i=1;i<=k;i++) arr[i]=tot[dom,i]; for(i=1;i<=k;i++) for(j=i+1;j<=k;j++) if(arr[i]>arr[j]){tmp=arr[i]; arr[i]=arr[j]; arr[j]=tmp} idx=int(0.95*k); if((0.95*k)>idx) idx++; if(idx<1) idx=1; if(idx>k) idx=k; p95=arr[idx]; print dom"\t"s"\t"tot[dom,k]"\t"rx[dom]"\t"tx[dom]"\t"p95"\t"rxTot[dom]"\t"txTot[dom]}}' "$states" "$samples" > "$merged"`,
+			`sort -t "$(printf '\t')" -k3,3nr -k1,1 "$merged" | awk -F'\t' 'BEGIN{printf "%-34s %-12s %10s %10s %10s %10s %11s %11s\n","DOMAIN","STATE","NET_Mbps","RX_Mbps","TX_Mbps","P95_Mbps","RX_TOTAL","TX_TOTAL"} {dom=fit($1,34); st=fit($2,12); net=mbpsOrDash($3); rxm=mbpsOrDash($4); txm=mbpsOrDash($5); p95=mbpsOrDash($6); r=humanOrDash($7); t=humanOrDash($8); printf "%-34s %-12s %10s %10s %10s %10s %11s %11s\n",dom,st,net,rxm,txm,p95,r,t} function mbpsOrDash(v){if(v==""||v<0)return "-"; return sprintf("%.2f",v+0)} function humanOrDash(v){if(v==""||v<0)return "-"; return human(v+0)} function human(n, u){if(n==0)return "0B"; if(n<0)return "-"; split("B KiB MiB GiB TiB PiB",u," "); i=1; while(n>=1024 && i<6){n/=1024; i++} if(n>=10 || i==1) return sprintf("%.0f%s",n,u[i]); return sprintf("%.1f%s",n,u[i]);} function fit(s,w){if(length(s)<=w)return s; return substr(s,1,w-1)"…"}'`,
+		}, "; "), nil
+	default:
+		return "", fmt.Errorf("unsupported kvm action %q", action)
+	}
+}
+
+func lxcPluginScript(action string, args []string) (string, error) {
+	mustName := func() (string, error) {
+		if len(args) < 1 || strings.TrimSpace(args[0]) == "" {
+			return "", errors.New("instance name is required")
+		}
+		return shellQuote(strings.TrimSpace(args[0])), nil
+	}
+
+	switch action {
+	case "", "list", "ls", "status":
+		return strings.Join([]string{
+			`printf "INSTANCE\tSTATE\tTYPE\tIPV4\tSNAPSHOTS\n"`,
+			`lxc list --format csv -c ns4tS 2>/dev/null | awk -F, 'NF>=1 {name=$1;state=$2;ipv4=$3;typ=$4;snap=$5; if(name=="") next; if(state=="") state="-"; if(typ=="") typ="-"; if(ipv4=="") ipv4="-"; if(snap=="") snap="0"; printf "%s\t%s\t%s\t%s\t%s\n", name, state, typ, ipv4, snap}'`,
+		}, "; "), nil
+	case "start":
+		n, err := mustName()
+		if err != nil {
+			return "", err
+		}
+		return "lxc start " + n, nil
+	case "stop":
+		n, err := mustName()
+		if err != nil {
+			return "", err
+		}
+		return "lxc stop " + n, nil
+	case "restart", "reboot":
+		n, err := mustName()
+		if err != nil {
+			return "", err
+		}
+		return "lxc restart " + n, nil
+	case "top":
+		return strings.Join([]string{
+			`printf "INSTANCE\tSTATE\tCPU_SEC\tMEM_CUR\tMEM_PEAK\tRX\tTX\n"`,
+			`lxc list --format csv -c ns 2>/dev/null | while IFS=, read -r name state; do [ -z "$name" ] && continue; info=$(lxc info "$name" --resources 2>/dev/null || lxc info "$name" 2>/dev/null || true); cpu=$(printf "%s\n" "$info" | awk -F': ' '/CPU usage \(in seconds\)/{print $2; exit}'); mem_cur=$(printf "%s\n" "$info" | awk -F': ' '/Memory \(current\)/{print $2; exit}'); mem_peak=$(printf "%s\n" "$info" | awk -F': ' '/Memory \(peak\)/{print $2; exit}'); rx=$(printf "%s\n" "$info" | awk -F': ' '/Bytes received/{print $2; exit}'); tx=$(printf "%s\n" "$info" | awk -F': ' '/Bytes sent/{print $2; exit}'); [ -z "$cpu" ] && cpu=0; [ -z "$mem_cur" ] && mem_cur="-"; [ -z "$mem_peak" ] && mem_peak="-"; [ -z "$rx" ] && rx="-"; [ -z "$tx" ] && tx="-"; printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n" "$name" "${state:--}" "$cpu" "$mem_cur" "$mem_peak" "$rx" "$tx"; done | sort -k3nr`,
+		}, "; "), nil
+	case "net-top", "net":
+		return strings.Join([]string{
+			`printf "INSTANCE\tSTATE\tRX_MiB\tTX_MiB\tTOTAL_MiB\n"`,
+			`lxc list --format csv -c ns 2>/dev/null | while IFS=, read -r name state; do [ -z "$name" ] && continue; raw=$(lxc query "/1.0/instances/$name/state" 2>/dev/null || true); rx=$(printf "%s\n" "$raw" | awk -F: '/"bytes_received"[[:space:]]*:/{gsub(/[^0-9]/,"",$2); s+=$2} END{printf "%.0f", s+0}'); tx=$(printf "%s\n" "$raw" | awk -F: '/"bytes_sent"[[:space:]]*:/{gsub(/[^0-9]/,"",$2); s+=$2} END{printf "%.0f", s+0}'); [ -z "$rx" ] && rx=0; [ -z "$tx" ] && tx=0; total=$((rx+tx)); rx_mib=$(awk -v n="$rx" 'BEGIN{printf "%.2f", n/1048576}'); tx_mib=$(awk -v n="$tx" 'BEGIN{printf "%.2f", n/1048576}'); total_mib=$(awk -v n="$total" 'BEGIN{printf "%.2f", n/1048576}'); printf "%s\t%s\t%s\t%s\t%s\n" "$name" "${state:--}" "$rx_mib" "$tx_mib" "$total_mib"; done | sort -k5nr`,
+		}, "; "), nil
+	case "stats", "info", "show":
+		n, err := mustName()
+		if err != nil {
+			return "", err
+		}
+		return "lxc info " + n + " --resources 2>/dev/null || lxc info " + n, nil
+	default:
+		return "", fmt.Errorf("unsupported lxc action %q", action)
+	}
+}
+
+func birdPluginScript(action string) (string, error) {
+	switch action {
+	case "", "status":
+		return "birdc show status", nil
+	case "protocols", "proto":
+		return "birdc show protocols", nil
+	case "routes", "route":
+		return "birdc show route", nil
+	default:
+		return "", fmt.Errorf("unsupported bird action %q", action)
+	}
+}
+
+func frrPluginScript(action string) (string, error) {
+	switch action {
+	case "", "status", "summary":
+		return "vtysh -c 'show version' -c 'show ip bgp summary'", nil
+	case "routes", "route":
+		return "vtysh -c 'show ip route summary' -c 'show ip route'", nil
+	case "bgp":
+		return "vtysh -c 'show ip bgp summary' -c 'show bgp ipv4 unicast summary'", nil
+	case "ospf":
+		return "vtysh -c 'show ip ospf neighbor' -c 'show ip ospf route'", nil
+	default:
+		return "", fmt.Errorf("unsupported frr action %q", action)
+	}
+}
+
+func shellQuote(v string) string {
+	return "'" + strings.ReplaceAll(v, "'", `'"'"'`) + "'"
+}
diff --git a/internal/cluster/service_test.go b/internal/cluster/service_test.go
new file mode 100644
index 0000000..3854e29
--- /dev/null
+++ b/internal/cluster/service_test.go
@@ -0,0 +1,554 @@
+package cluster
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func newTestService(t *testing.T) *Service {
+	t.Helper()
+
+	store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
+	if err != nil {
+		t.Fatalf("NewStore error: %v", err)
+	}
+
+	return NewService(store)
+}
+
+func TestConnectListUseDisconnect(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	ctx := context.Background()
+
+	c1, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "eu-1",
+		Host:       "10.0.0.10",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass1",
+		SkipCheck:  true,
+	})
+	if err != nil {
+		t.Fatalf("connect c1 error: %v", err)
+	}
+
+	c2, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "us-1",
+		Host:       "10.0.0.11",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass2",
+		SkipCheck:  true,
+	})
+	if err != nil {
+		t.Fatalf("connect c2 error: %v", err)
+	}
+
+	clusters, activeID, err := svc.List()
+	if err != nil {
+		t.Fatalf("list error: %v", err)
+	}
+	if len(clusters) != 2 {
+		t.Fatalf("expected 2 clusters, got %d", len(clusters))
+	}
+	if activeID != c2.ID {
+		t.Fatalf("expected active %s, got %s", c2.ID, activeID)
+	}
+
+	_, err = svc.Use(c1.Name)
+	if err != nil {
+		t.Fatalf("use error: %v", err)
+	}
+
+	current, err := svc.Current()
+	if err != nil {
+		t.Fatalf("current error: %v", err)
+	}
+	if current.ID != c1.ID {
+		t.Fatalf("expected current %s, got %s", c1.ID, current.ID)
+	}
+
+	_, err = svc.Disconnect(c1.Name)
+	if err != nil {
+		t.Fatalf("disconnect error: %v", err)
+	}
+
+	current, err = svc.Current()
+	if err != nil {
+		t.Fatalf("current after disconnect error: %v", err)
+	}
+	if current.ID != c2.ID {
+		t.Fatalf("expected fallback current %s, got %s", c2.ID, current.ID)
+	}
+}
+
+func TestConnectDuplicateNameRequiresForce(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	ctx := context.Background()
+
+	_, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "prod",
+		Host:       "10.0.0.10",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass1",
+		SkipCheck:  true,
+	})
+	if err != nil {
+		t.Fatalf("initial connect error: %v", err)
+	}
+
+	_, _, err = svc.Connect(ctx, ConnectOptions{
+		Name:       "prod",
+		Host:       "10.0.0.20",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass2",
+		SkipCheck:  true,
+	})
+	if err == nil {
+		t.Fatal("expected duplicate name error")
+	}
+
+	c, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "prod",
+		Host:       "10.0.0.20",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass2",
+		SkipCheck:  true,
+		Force:      true,
+	})
+	if err != nil {
+		t.Fatalf("force connect error: %v", err)
+	}
+	if c.Host != "10.0.0.20" {
+		t.Fatalf("expected overwritten host, got %s", c.Host)
+	}
+}
+
+func TestExportImportRestoresKeyFilesAndNewFields(t *testing.T) {
+	t.Parallel()
+
+	srcDir := t.TempDir()
+	keyPath := filepath.Join(srcDir, "id_ed25519")
+	passPath := filepath.Join(srcDir, "pass.pxmonpassphrase")
+	sftpKeyPath := filepath.Join(srcDir, "sftp_key")
+	if err := os.WriteFile(keyPath, []byte("PRIVATE KEY\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(passPath, []byte("secret-pass\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(sftpKeyPath, []byte("SFTP PRIVATE KEY\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	src := newTestService(t)
+	reg := newRegistry()
+	reg.ActiveClusterID = "clu_1"
+	reg.Backups.Targets = []BackupTarget{{
+		ID:          "bt_1",
+		Name:        "sftp",
+		Type:        "sftp",
+		Enabled:     true,
+		SFTPKeyPath: sftpKeyPath,
+	}}
+	reg.Clusters = []Cluster{{
+		ID:                "clu_1",
+		Name:              "node",
+		Host:              "192.0.2.10",
+		Port:              22,
+		User:              "root",
+		Transport:         TransportIPFabric,
+		AuthMethod:        AuthMethodKey,
+		KeyPath:           keyPath,
+		KeyPassphraseFile: passPath,
+		RepoTunnel: RepoTunnelState{
+			Enabled: true,
+			Proxy:   "http://203.0.113.10:3128",
+			Source:  "dnf",
+		},
+		Alerts:    defaultAlertPolicy(),
+		CreatedAt: time.Now().UTC(),
+		UpdatedAt: time.Now().UTC(),
+	}}
+	if err := src.store.Save(reg); err != nil {
+		t.Fatalf("save source registry: %v", err)
+	}
+
+	bundlePath := filepath.Join(t.TempDir(), "pxmon-export.enc")
+	if err := src.Export(bundlePath, "strong-test-passphrase"); err != nil {
+		t.Fatalf("export error: %v", err)
+	}
+	raw, err := os.ReadFile(bundlePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(string(raw), "PRIVATE KEY") || strings.Contains(string(raw), "secret-pass") {
+		t.Fatal("export bundle leaked key material in plaintext")
+	}
+
+	dst := newTestService(t)
+	if _, err := dst.Import(bundlePath, "strong-test-passphrase", ImportModeReplace); err != nil {
+		t.Fatalf("import error: %v", err)
+	}
+	gotReg, err := dst.store.Load()
+	if err != nil {
+		t.Fatalf("load imported registry: %v", err)
+	}
+	if len(gotReg.Clusters) != 1 {
+		t.Fatalf("expected 1 cluster, got %d", len(gotReg.Clusters))
+	}
+	got := gotReg.Clusters[0]
+	if !got.RepoTunnel.Enabled || got.RepoTunnel.Proxy != "http://203.0.113.10:3128" {
+		t.Fatalf("repo tunnel was not preserved: %+v", got.RepoTunnel)
+	}
+	if got.KeyPath == keyPath || got.KeyPassphraseFile == passPath {
+		t.Fatalf("expected key paths to be restored under destination config dir, got key=%q pass=%q", got.KeyPath, got.KeyPassphraseFile)
+	}
+	keyData, err := os.ReadFile(got.KeyPath)
+	if err != nil {
+		t.Fatalf("read restored key: %v", err)
+	}
+	if string(keyData) != "PRIVATE KEY\n" {
+		t.Fatalf("unexpected restored key content: %q", string(keyData))
+	}
+	passData, err := os.ReadFile(got.KeyPassphraseFile)
+	if err != nil {
+		t.Fatalf("read restored passphrase file: %v", err)
+	}
+	if string(passData) != "secret-pass\n" {
+		t.Fatalf("unexpected restored passphrase content: %q", string(passData))
+	}
+	if len(gotReg.Backups.Targets) != 1 || gotReg.Backups.Targets[0].SFTPKeyPath == sftpKeyPath {
+		t.Fatalf("expected restored SFTP key path, got %+v", gotReg.Backups.Targets)
+	}
+
+	mergeDst := newTestService(t)
+	if _, err := mergeDst.Import(bundlePath, "strong-test-passphrase", ImportModeMerge); err != nil {
+		t.Fatalf("merge import error: %v", err)
+	}
+	mergeReg, err := mergeDst.store.Load()
+	if err != nil {
+		t.Fatalf("load merge registry: %v", err)
+	}
+	if len(mergeReg.Backups.Targets) != 1 || strings.TrimSpace(mergeReg.Backups.Targets[0].SFTPKeyPath) == "" {
+		t.Fatalf("expected backup target to be merged, got %+v", mergeReg.Backups)
+	}
+}
+
+func TestConnectRequiresAuthData(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	ctx := context.Background()
+
+	_, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "bad",
+		Host:       "10.0.0.10",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		SkipCheck:  true,
+	})
+	if err == nil {
+		t.Fatal("expected error for empty password auth")
+	}
+}
+
+func TestAlertPolicySetAndGet(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	ctx := context.Background()
+
+	_, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "node-1",
+		Host:       "10.0.0.10",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass",
+		SkipCheck:  true,
+	})
+	if err != nil {
+		t.Fatalf("connect error: %v", err)
+	}
+
+	updated, err := svc.SetAlertPolicy("node-1", AlertPolicy{
+		CPUWarnPercent:         70,
+		RAMWarnPercent:         75,
+		SwapWarnPercent:        60,
+		DiskWarnPercent:        80,
+		NetWarnMbps:            120,
+		NetSustainEnabled:      true,
+		NetSustainIface:        "eth0",
+		NetSustainInclude:      []string{"net0"},
+		NetSustainExclude:      []string{"backup"},
+		NetSustainMbps:         500,
+		NetSustainMinutes:      60,
+		NetSustainCooldownMins: 15,
+	})
+	if err != nil {
+		t.Fatalf("set alert policy error: %v", err)
+	}
+
+	if updated.Alerts.NetWarnMbps != 120 {
+		t.Fatalf("unexpected net threshold: %.2f", updated.Alerts.NetWarnMbps)
+	}
+
+	got, err := svc.GetAlertPolicy("node-1")
+	if err != nil {
+		t.Fatalf("get alert policy error: %v", err)
+	}
+	if got.RAMWarnPercent != 75 || got.DiskWarnPercent != 80 {
+		t.Fatalf("unexpected thresholds: %+v", got)
+	}
+	if !got.NetSustainEnabled || got.NetSustainIface != "eth0" || got.NetSustainMbps != 500 {
+		t.Fatalf("unexpected sustained net policy: %+v", got)
+	}
+	if len(got.NetSustainInclude) != 1 || got.NetSustainInclude[0] != "net0" {
+		t.Fatalf("unexpected sustained net include filter: %+v", got.NetSustainInclude)
+	}
+	if len(got.NetSustainExclude) != 1 || got.NetSustainExclude[0] != "backup" {
+		t.Fatalf("unexpected sustained net exclude filter: %+v", got.NetSustainExclude)
+	}
+}
+
+func TestIsPluginToolSupported(t *testing.T) {
+	t.Parallel()
+
+	info := SoftwareInfo{
+		Bird: true,
+		FRR:  true,
+		KVM:  true,
+		LXC:  true,
+		LXD:  false,
+	}
+
+	tests := []struct {
+		tool string
+		ok   bool
+	}{
+		{tool: "bird", ok: true},
+		{tool: "frr", ok: true},
+		{tool: "kvm", ok: true},
+		{tool: "lxc", ok: true},
+		{tool: "lxd", ok: true}, // lxd aliases to lxc command templates
+		{tool: "unknown", ok: false},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.tool, func(t *testing.T) {
+			t.Parallel()
+			if got := isPluginToolSupported(info, tt.tool); got != tt.ok {
+				t.Fatalf("tool=%s expected %v got %v", tt.tool, tt.ok, got)
+			}
+		})
+	}
+}
+
+func TestLXDTopUsesTemplateNotRawLxcTop(t *testing.T) {
+	t.Parallel()
+
+	script, err := pluginScript("lxd", "top", nil)
+	if err != nil {
+		t.Fatalf("pluginScript error: %v", err)
+	}
+	if strings.Contains(script, "lxc top") {
+		t.Fatalf("expected custom template script, got raw lxc top: %q", script)
+	}
+	if !strings.Contains(script, "lxc info") {
+		t.Fatalf("expected lxc info usage in top template")
+	}
+}
+
+func TestRunPluginActionReportsMissingSupport(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	ctx := context.Background()
+
+	cluster, _, err := svc.Connect(ctx, ConnectOptions{
+		Name:       "eu-1",
+		Host:       "127.0.0.1",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "pass",
+		SkipCheck:  true,
+	})
+	if err != nil {
+		t.Fatalf("connect error: %v", err)
+	}
+
+	reg, err := svc.store.Load()
+	if err != nil {
+		t.Fatalf("load registry error: %v", err)
+	}
+	for i := range reg.Clusters {
+		if reg.Clusters[i].ID == cluster.ID {
+			reg.Clusters[i].Software = SoftwareInfo{
+				DetectedAt: time.Now().UTC(),
+			}
+			break
+		}
+	}
+	if err := svc.store.Save(reg); err != nil {
+		t.Fatalf("save registry error: %v", err)
+	}
+
+	_, err = svc.RunPluginAction(ctx, cluster.Name, "lxd", "top", nil)
+	if err == nil {
+		t.Fatalf("expected unsupported software error")
+	}
+	if !strings.Contains(err.Error(), "support for lxd was not detected") {
+		t.Fatalf("unexpected error message: %v", err)
+	}
+}
+
+func TestKVMTopScriptIncludesReadableMetrics(t *testing.T) {
+	t.Parallel()
+
+	script, err := pluginScript("kvm", "top", nil)
+	if err != nil {
+		t.Fatalf("pluginScript error: %v", err)
+	}
+	for _, want := range []string{
+		"VCPU",
+		"RAM_MAX",
+		"DISK_CAP",
+		"DISK_ALLOC",
+	} {
+		if !strings.Contains(script, want) {
+			t.Fatalf("expected %q in kvm top script", want)
+		}
+	}
+}
+
+func TestKVMNetTopScriptIncludesRateAndP95(t *testing.T) {
+	t.Parallel()
+
+	script, err := pluginScript("kvm", "net-top", nil)
+	if err != nil {
+		t.Fatalf("pluginScript error: %v", err)
+	}
+	for _, want := range []string{
+		"NET_Mbps",
+		"RX_Mbps",
+		"TX_Mbps",
+		"P95_Mbps",
+		"RX_TOTAL",
+		"TX_TOTAL",
+	} {
+		if !strings.Contains(script, want) {
+			t.Fatalf("expected %q in kvm net-top script", want)
+		}
+	}
+}
+
+func TestKVMScriptsAreShellParseable(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		tool   string
+		action string
+	}{
+		{tool: "kvm", action: "top"},
+		{tool: "kvm", action: "net-top"},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.tool+"-"+tc.action, func(t *testing.T) {
+			t.Parallel()
+
+			script, err := pluginScript(tc.tool, tc.action, nil)
+			if err != nil {
+				t.Fatalf("pluginScript error: %v", err)
+			}
+
+			cmd := exec.Command("sh", "-n", "-c", script)
+			out, err := cmd.CombinedOutput()
+			if err != nil {
+				t.Fatalf("shell parse failed: %v\n%s\nSCRIPT:\n%s", err, string(out), script)
+			}
+		})
+	}
+}
+
+func TestTelegramConfigSetGetDisable(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	cfg, err := svc.SetTelegram(Telegram{
+		Enabled:        true,
+		Token:          "123:ABC",
+		AllowedUserIDs: []int64{2002, 1001, 2002},
+	})
+	if err != nil {
+		t.Fatalf("set telegram config: %v", err)
+	}
+	if !cfg.Enabled {
+		t.Fatal("expected enabled telegram config")
+	}
+	if len(cfg.AllowedUserIDs) != 2 {
+		t.Fatalf("expected deduped ids, got %+v", cfg.AllowedUserIDs)
+	}
+
+	got, err := svc.GetTelegram()
+	if err != nil {
+		t.Fatalf("get telegram config: %v", err)
+	}
+	if got.Token != "123:ABC" {
+		t.Fatalf("unexpected token: %q", got.Token)
+	}
+	if len(got.AllowedUserIDs) != 2 || got.AllowedUserIDs[0] != 1001 || got.AllowedUserIDs[1] != 2002 {
+		t.Fatalf("unexpected allowed ids: %+v", got.AllowedUserIDs)
+	}
+
+	disabled, err := svc.DisableTelegram()
+	if err != nil {
+		t.Fatalf("disable telegram config: %v", err)
+	}
+	if disabled.Enabled {
+		t.Fatal("expected disabled telegram config")
+	}
+}
+
+func TestTelegramConfigValidation(t *testing.T) {
+	t.Parallel()
+
+	svc := newTestService(t)
+	_, err := svc.SetTelegram(Telegram{
+		Enabled:        true,
+		AllowedUserIDs: []int64{123},
+	})
+	if err == nil {
+		t.Fatal("expected token validation error")
+	}
+
+	_, err = svc.SetTelegram(Telegram{
+		Enabled: true,
+		Token:   "123:ABC",
+	})
+	if err == nil {
+		t.Fatal("expected allowed ids validation error")
+	}
+}
diff --git a/internal/cluster/slo_capacity.go b/internal/cluster/slo_capacity.go
new file mode 100644
index 0000000..5d2d53b
--- /dev/null
+++ b/internal/cluster/slo_capacity.go
@@ -0,0 +1,159 @@
+package cluster
+
+import (
+	"math"
+	"sort"
+	"strings"
+	"time"
+
+	"pxmon/internal/history"
+)
+
+type AvailabilityVM struct {
+	Name         string  `json:"name"`
+	Availability float64 `json:"availability_pct"`
+	Samples      int     `json:"samples"`
+	Running      int     `json:"running_samples"`
+}
+
+type AvailabilityReport struct {
+	Cluster      string           `json:"cluster"`
+	Range        string           `json:"range"`
+	Samples      int              `json:"samples"`
+	UpSamples    int              `json:"up_samples"`
+	Availability float64          `json:"availability_pct"`
+	VMs          []AvailabilityVM `json:"vms,omitempty"`
+}
+
+func (s *Service) AvailabilityReport(selector string, since time.Time, vmFilter string) (AvailabilityReport, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return AvailabilityReport{}, err
+	}
+	store := history.NewAvailabilityStore(s.DataDir())
+	snaps, err := store.Load(c.ID, since)
+	if err != nil {
+		return AvailabilityReport{}, err
+	}
+	rep := AvailabilityReport{Cluster: c.Name, Samples: len(snaps)}
+	if len(snaps) == 0 {
+		return rep, nil
+	}
+	vmFilter = strings.TrimSpace(vmFilter)
+	totalUp := 0
+	type acc struct{ samples, running int }
+	vmap := map[string]*acc{}
+	for _, snap := range snaps {
+		if snap.ClusterUp {
+			totalUp++
+		}
+		for vm, st := range snap.VMStates {
+			if vmFilter != "" && !strings.EqualFold(vmFilter, vm) {
+				continue
+			}
+			a := vmap[vm]
+			if a == nil {
+				a = &acc{}
+				vmap[vm] = a
+			}
+			a.samples++
+			if strings.EqualFold(strings.TrimSpace(st), "running") {
+				a.running++
+			}
+		}
+	}
+	rep.UpSamples = totalUp
+	rep.Availability = 100 * float64(totalUp) / float64(len(snaps))
+	for vm, a := range vmap {
+		if a.samples == 0 {
+			continue
+		}
+		rep.VMs = append(rep.VMs, AvailabilityVM{
+			Name:         vm,
+			Samples:      a.samples,
+			Running:      a.running,
+			Availability: 100 * float64(a.running) / float64(a.samples),
+		})
+	}
+	sort.Slice(rep.VMs, func(i, j int) bool { return rep.VMs[i].Name < rep.VMs[j].Name })
+	return rep, nil
+}
+
+type CapacityForecastItem struct {
+	Mount         string  `json:"mount"`
+	UsedPct       float64 `json:"used_pct"`
+	SlopeBytesSec float64 `json:"slope_bytes_per_sec"`
+	DaysTo90      float64 `json:"days_to_90_pct"`
+	DaysTo95      float64 `json:"days_to_95_pct"`
+}
+
+type CapacityForecastReport struct {
+	Cluster string                 `json:"cluster"`
+	Samples int                    `json:"samples"`
+	Items   []CapacityForecastItem `json:"items,omitempty"`
+}
+
+func (s *Service) CapacityForecast(selector string, since time.Time) (CapacityForecastReport, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return CapacityForecastReport{}, err
+	}
+	store := history.NewCapacityStore(s.DataDir())
+	snaps, err := store.Load(c.ID, since)
+	if err != nil {
+		return CapacityForecastReport{}, err
+	}
+	rep := CapacityForecastReport{Cluster: c.Name, Samples: len(snaps)}
+	if len(snaps) < 2 {
+		return rep, nil
+	}
+	type point struct {
+		ts   time.Time
+		used float64
+		tot  float64
+	}
+	byMount := map[string][]point{}
+	for _, snap := range snaps {
+		for _, d := range snap.Disks {
+			if d.TotalBytes == 0 {
+				continue
+			}
+			byMount[d.Mount] = append(byMount[d.Mount], point{ts: snap.Timestamp, used: float64(d.UsedBytes), tot: float64(d.TotalBytes)})
+		}
+	}
+	for mnt, pts := range byMount {
+		if len(pts) < 2 {
+			continue
+		}
+		sort.Slice(pts, func(i, j int) bool { return pts[i].ts.Before(pts[j].ts) })
+		first := pts[0]
+		last := pts[len(pts)-1]
+		dt := last.ts.Sub(first.ts).Seconds()
+		if dt <= 0 {
+			continue
+		}
+		slope := (last.used - first.used) / dt
+		usedPct := 100 * last.used / last.tot
+		d90 := daysToTarget(last.used, last.tot*0.90, slope)
+		d95 := daysToTarget(last.used, last.tot*0.95, slope)
+		rep.Items = append(rep.Items, CapacityForecastItem{
+			Mount:         mnt,
+			UsedPct:       usedPct,
+			SlopeBytesSec: slope,
+			DaysTo90:      d90,
+			DaysTo95:      d95,
+		})
+	}
+	sort.Slice(rep.Items, func(i, j int) bool { return rep.Items[i].UsedPct > rep.Items[j].UsedPct })
+	return rep, nil
+}
+
+func daysToTarget(current, target, slope float64) float64 {
+	if target <= current {
+		return 0
+	}
+	if slope <= 0 {
+		return math.Inf(1)
+	}
+	return (target - current) / slope / 86400
+}
diff --git a/internal/cluster/store.go b/internal/cluster/store.go
new file mode 100644
index 0000000..a03a034
--- /dev/null
+++ b/internal/cluster/store.go
@@ -0,0 +1,425 @@
+package cluster
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	envConfigPath    = "PXMON_CONFIG"
+	envMasterKeyPath = "PXMON_MASTER_KEY"
+	filePrefix       = "OBSCTL1:"
+	masterKeyBytes   = 32
+)
+
+// Store persists encrypted cluster registry on disk.
+type Store struct {
+	path    string
+	keyPath string
+}
+
+func NewStore(path string) (*Store, error) {
+	if path == "" {
+		var err error
+		path, err = DefaultConfigPath()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	keyPath, err := defaultMasterKeyPath(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Store{path: path, keyPath: keyPath}, nil
+}
+
+func DefaultConfigPath() (string, error) {
+	if p := os.Getenv(envConfigPath); p != "" {
+		return p, nil
+	}
+
+	dir, err := os.UserConfigDir()
+	if err != nil {
+		return "", fmt.Errorf("resolve user config dir: %w", err)
+	}
+
+	return filepath.Join(dir, "pxmon", "clusters.enc"), nil
+}
+
+func defaultMasterKeyPath(configPath string) (string, error) {
+	if p := os.Getenv(envMasterKeyPath); p != "" {
+		return p, nil
+	}
+	if configPath == "" {
+		return "", errors.New("empty config path")
+	}
+	return filepath.Join(filepath.Dir(configPath), "master.key"), nil
+}
+
+func (s *Store) Path() string {
+	return s.path
+}
+
+func (s *Store) KeyPath() string {
+	return s.keyPath
+}
+
+func (s *Store) LockerSessionPath() string {
+	return filepath.Join(filepath.Dir(s.path), "locker.session")
+}
+
+func (s *Store) LockerAuditPath() string {
+	return filepath.Join(filepath.Dir(s.path), "locker.audit.log")
+}
+
+func (s *Store) Load() (Registry, error) {
+	f, err := os.Open(s.path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return newRegistry(), nil
+		}
+		return Registry{}, fmt.Errorf("open registry file: %w", err)
+	}
+	defer f.Close()
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return Registry{}, fmt.Errorf("read registry file: %w", err)
+	}
+	if len(strings.TrimSpace(string(data))) == 0 {
+		return newRegistry(), nil
+	}
+
+	payload, err := s.decodePayload(data)
+	if err != nil {
+		return Registry{}, err
+	}
+
+	var reg Registry
+	if err := json.Unmarshal(payload, ®); err != nil {
+		return Registry{}, fmt.Errorf("decode registry JSON: %w", err)
+	}
+
+	if reg.Version == 0 {
+		reg.Version = currentVersion
+	}
+	if reg.Clusters == nil {
+		reg.Clusters = []Cluster{}
+	}
+	for i := range reg.Clusters {
+		reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
+		reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
+		reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
+		reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
+		reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
+		reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
+		reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
+		reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
+	}
+	reg.Telegram = normalizeTelegram(reg.Telegram)
+	reg.Locker = normalizeLocker(reg.Locker)
+	reg.Backups = normalizeBackupConfig(reg.Backups)
+
+	return reg, nil
+}
+
+func (s *Store) decodePayload(data []byte) ([]byte, error) {
+	text := strings.TrimSpace(string(data))
+
+	if strings.HasPrefix(text, "{") {
+		// Backward compatibility with legacy unencrypted format.
+		return []byte(text), nil
+	}
+
+	if !strings.HasPrefix(text, filePrefix) {
+		return nil, errors.New("unsupported registry format")
+	}
+
+	blob := strings.TrimPrefix(text, filePrefix)
+	raw, err := base64.StdEncoding.DecodeString(blob)
+	if err != nil {
+		return nil, fmt.Errorf("decode encrypted payload: %w", err)
+	}
+
+	key, err := s.loadOrCreateMasterKey()
+	if err != nil {
+		return nil, err
+	}
+
+	payload, err := decrypt(raw, key)
+	if err != nil {
+		return nil, fmt.Errorf("decrypt registry: %w", err)
+	}
+	return payload, nil
+}
+
+func (s *Store) Save(reg Registry) error {
+	reg.Version = currentVersion
+	if reg.Clusters == nil {
+		reg.Clusters = []Cluster{}
+	}
+	for i := range reg.Clusters {
+		reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
+		reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
+		reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
+		reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
+		reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
+		reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
+		reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
+		reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
+	}
+	reg.Telegram = normalizeTelegram(reg.Telegram)
+	reg.Locker = normalizeLocker(reg.Locker)
+	reg.Backups = normalizeBackupConfig(reg.Backups)
+
+	if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
+		return fmt.Errorf("create config dir: %w", err)
+	}
+
+	payload, err := json.MarshalIndent(reg, "", "  ")
+	if err != nil {
+		return fmt.Errorf("encode registry JSON: %w", err)
+	}
+
+	key, err := s.loadOrCreateMasterKey()
+	if err != nil {
+		return err
+	}
+
+	encrypted, err := encrypt(payload, key)
+	if err != nil {
+		return fmt.Errorf("encrypt registry: %w", err)
+	}
+
+	content := filePrefix + base64.StdEncoding.EncodeToString(encrypted) + "\n"
+
+	tmp := s.path + ".tmp"
+	if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
+		return fmt.Errorf("write temp registry file: %w", err)
+	}
+
+	if err := os.Rename(tmp, s.path); err != nil {
+		return fmt.Errorf("replace registry file: %w", err)
+	}
+
+	return nil
+}
+
+func (s *Store) loadOrCreateMasterKey() ([]byte, error) {
+	if err := os.MkdirAll(filepath.Dir(s.keyPath), 0o700); err != nil {
+		return nil, fmt.Errorf("create key dir: %w", err)
+	}
+
+	key, err := os.ReadFile(s.keyPath)
+	if err == nil {
+		if len(key) != masterKeyBytes {
+			return nil, fmt.Errorf("invalid master key length: got %d", len(key))
+		}
+		return key, nil
+	}
+	if !errors.Is(err, os.ErrNotExist) {
+		return nil, fmt.Errorf("read master key: %w", err)
+	}
+
+	key = make([]byte, masterKeyBytes)
+	if _, err := rand.Read(key); err != nil {
+		return nil, fmt.Errorf("generate master key: %w", err)
+	}
+
+	tmp := s.keyPath + ".tmp"
+	if err := os.WriteFile(tmp, key, 0o600); err != nil {
+		return nil, fmt.Errorf("write temp master key: %w", err)
+	}
+	if err := os.Rename(tmp, s.keyPath); err != nil {
+		return nil, fmt.Errorf("replace master key: %w", err)
+	}
+
+	return key, nil
+}
+
+func encrypt(payload, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	nonce := make([]byte, gcm.NonceSize())
+	if _, err := rand.Read(nonce); err != nil {
+		return nil, err
+	}
+
+	sealed := gcm.Seal(nil, nonce, payload, nil)
+	out := make([]byte, 0, len(nonce)+len(sealed))
+	out = append(out, nonce...)
+	out = append(out, sealed...)
+	return out, nil
+}
+
+func decrypt(raw, key []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	nonceSize := gcm.NonceSize()
+	if len(raw) <= nonceSize {
+		return nil, errors.New("ciphertext too short")
+	}
+
+	nonce := raw[:nonceSize]
+	ciphertext := raw[nonceSize:]
+
+	payload, err := gcm.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return nil, err
+	}
+	return payload, nil
+}
+
+type lockerSessionState struct {
+	ExpiresAtUnix int64  `json:"expires_at_unix"`
+	SigHex        string `json:"sig_hex"`
+}
+
+func (s *Store) SaveLockerSession(passwordHash string, ttl time.Duration) error {
+	passwordHash = strings.TrimSpace(passwordHash)
+	if passwordHash == "" {
+		return errors.New("empty locker password hash")
+	}
+	if ttl <= 0 {
+		ttl = 6 * time.Hour
+	}
+
+	key, err := s.loadOrCreateMasterKey()
+	if err != nil {
+		return err
+	}
+
+	expires := time.Now().UTC().Add(ttl).Unix()
+	payload := strconv.FormatInt(expires, 10) + "|" + passwordHash
+	mac := hmac.New(sha256.New, key)
+	_, _ = mac.Write([]byte(payload))
+	sig := hex.EncodeToString(mac.Sum(nil))
+
+	state := lockerSessionState{
+		ExpiresAtUnix: expires,
+		SigHex:        sig,
+	}
+	data, err := json.Marshal(state)
+	if err != nil {
+		return err
+	}
+	data = append(data, '\n')
+
+	path := s.LockerSessionPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	tmp := path + ".tmp"
+	if err := os.WriteFile(tmp, data, 0o600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+func (s *Store) ValidateLockerSession(passwordHash string) (bool, time.Time, error) {
+	passwordHash = strings.TrimSpace(passwordHash)
+	if passwordHash == "" {
+		return false, time.Time{}, nil
+	}
+
+	path := s.LockerSessionPath()
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return false, time.Time{}, nil
+		}
+		return false, time.Time{}, err
+	}
+
+	var state lockerSessionState
+	if err := json.Unmarshal(raw, &state); err != nil {
+		return false, time.Time{}, nil
+	}
+	if state.ExpiresAtUnix <= 0 || strings.TrimSpace(state.SigHex) == "" {
+		return false, time.Time{}, nil
+	}
+	expiresAt := time.Unix(state.ExpiresAtUnix, 0).UTC()
+	if time.Now().UTC().After(expiresAt) {
+		return false, expiresAt, nil
+	}
+
+	key, err := s.loadOrCreateMasterKey()
+	if err != nil {
+		return false, time.Time{}, err
+	}
+
+	payload := strconv.FormatInt(state.ExpiresAtUnix, 10) + "|" + passwordHash
+	mac := hmac.New(sha256.New, key)
+	_, _ = mac.Write([]byte(payload))
+	expected := mac.Sum(nil)
+	got, err := hex.DecodeString(strings.TrimSpace(state.SigHex))
+	if err != nil {
+		return false, expiresAt, nil
+	}
+	if !hmac.Equal(expected, got) {
+		return false, expiresAt, nil
+	}
+	return true, expiresAt, nil
+}
+
+func (s *Store) ClearLockerSession() error {
+	path := s.LockerSessionPath()
+	if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	return nil
+}
+
+func (s *Store) AppendLockerAudit(event, detail string) error {
+	path := s.LockerAuditPath()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	ts := time.Now().UTC().Format(time.RFC3339)
+	event = strings.TrimSpace(event)
+	detail = strings.TrimSpace(detail)
+	if event == "" {
+		event = "event"
+	}
+	if detail == "" {
+		detail = "-"
+	}
+	_, err = fmt.Fprintf(f, "%s event=%s detail=%s\n", ts, event, strconv.Quote(detail))
+	return err
+}
diff --git a/internal/cluster/store_test.go b/internal/cluster/store_test.go
new file mode 100644
index 0000000..8286f87
--- /dev/null
+++ b/internal/cluster/store_test.go
@@ -0,0 +1,110 @@
+package cluster
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestStoreLoadMissing(t *testing.T) {
+	t.Parallel()
+
+	store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
+	if err != nil {
+		t.Fatalf("NewStore error: %v", err)
+	}
+
+	reg, err := store.Load()
+	if err != nil {
+		t.Fatalf("Load error: %v", err)
+	}
+
+	if reg.Version != currentVersion {
+		t.Fatalf("expected version %d, got %d", currentVersion, reg.Version)
+	}
+	if len(reg.Clusters) != 0 {
+		t.Fatalf("expected empty clusters, got %d", len(reg.Clusters))
+	}
+}
+
+func TestStoreSaveLoadRoundtrip(t *testing.T) {
+	t.Parallel()
+
+	path := filepath.Join(t.TempDir(), "clusters.enc")
+	store, err := NewStore(path)
+	if err != nil {
+		t.Fatalf("NewStore error: %v", err)
+	}
+
+	seed := newRegistry()
+	seed.ActiveClusterID = "clu_1"
+	seed.Clusters = []Cluster{{
+		ID:         "clu_1",
+		Name:       "prod",
+		Host:       "10.0.0.10",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "secret123",
+	}}
+
+	if err := store.Save(seed); err != nil {
+		t.Fatalf("Save error: %v", err)
+	}
+
+	got, err := store.Load()
+	if err != nil {
+		t.Fatalf("Load error: %v", err)
+	}
+
+	if got.ActiveClusterID != "clu_1" {
+		t.Fatalf("active cluster mismatch: %s", got.ActiveClusterID)
+	}
+	if len(got.Clusters) != 1 || got.Clusters[0].Name != "prod" {
+		t.Fatalf("unexpected clusters: %+v", got.Clusters)
+	}
+	if got.Clusters[0].Password != "secret123" {
+		t.Fatalf("password mismatch after decrypt: %s", got.Clusters[0].Password)
+	}
+}
+
+func TestStorePersistsEncryptedPayload(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	path := filepath.Join(dir, "clusters.enc")
+	store, err := NewStore(path)
+	if err != nil {
+		t.Fatalf("NewStore error: %v", err)
+	}
+
+	reg := newRegistry()
+	reg.Clusters = []Cluster{{
+		ID:         "clu_1",
+		Name:       "sensitive-prod",
+		Host:       "192.168.1.1",
+		Port:       22,
+		User:       "root",
+		AuthMethod: AuthMethodPassword,
+		Password:   "very-secret",
+	}}
+	if err := store.Save(reg); err != nil {
+		t.Fatalf("Save error: %v", err)
+	}
+
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("ReadFile error: %v", err)
+	}
+	if !strings.HasPrefix(string(raw), filePrefix) {
+		t.Fatalf("expected encrypted file prefix %q", filePrefix)
+	}
+	if strings.Contains(string(raw), "sensitive-prod") || strings.Contains(string(raw), "very-secret") {
+		t.Fatal("plaintext secrets leaked into encrypted file")
+	}
+
+	if _, err := os.Stat(store.KeyPath()); err != nil {
+		t.Fatalf("master key not created: %v", err)
+	}
+}
diff --git a/internal/cluster/tags.go b/internal/cluster/tags.go
new file mode 100644
index 0000000..33db2fa
--- /dev/null
+++ b/internal/cluster/tags.go
@@ -0,0 +1,144 @@
+package cluster
+
+import (
+	"errors"
+	"sort"
+	"strings"
+)
+
+func (s *Service) AddClusterTags(selector string, tags []string) (Cluster, error) {
+	return s.updateClusterTags(selector, tags, true)
+}
+
+func (s *Service) RemoveClusterTags(selector string, tags []string) (Cluster, error) {
+	return s.updateClusterTags(selector, tags, false)
+}
+
+func (s *Service) updateClusterTags(selector string, tags []string, add bool) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	changes := make(map[string]struct{}, len(tags))
+	for _, t := range normalizeTagList(tags) {
+		changes[t] = struct{}{}
+	}
+	if len(changes) == 0 {
+		return c, errors.New("at least one non-empty tag is required")
+	}
+
+	current := make(map[string]struct{}, len(c.Tags))
+	for _, t := range normalizeTagList(c.Tags) {
+		current[t] = struct{}{}
+	}
+	if add {
+		for t := range changes {
+			current[t] = struct{}{}
+		}
+	} else {
+		for t := range changes {
+			delete(current, t)
+		}
+	}
+	out := make([]string, 0, len(current))
+	for t := range current {
+		out = append(out, t)
+	}
+	sort.Strings(out)
+	c.Tags = out
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("cluster.tags", c.Name, strings.Join(out, ","))
+	return c, nil
+}
+
+func (s *Service) ListClusterTags(selector string) ([]string, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return nil, err
+	}
+	return normalizeTagList(c.Tags), nil
+}
+
+func (s *Service) AddKVMTag(selector, vm string, tags []string) (Cluster, error) {
+	return s.updateKVMTags(selector, vm, tags, true)
+}
+
+func (s *Service) RemoveKVMTag(selector, vm string, tags []string) (Cluster, error) {
+	return s.updateKVMTags(selector, vm, tags, false)
+}
+
+func (s *Service) updateKVMTags(selector, vm string, tags []string, add bool) (Cluster, error) {
+	vm = strings.TrimSpace(vm)
+	if vm == "" {
+		return Cluster{}, errors.New("vm name is required")
+	}
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	if c.KVMTags == nil {
+		c.KVMTags = map[string][]string{}
+	}
+	current := make(map[string]struct{}, len(c.KVMTags[vm]))
+	for _, t := range normalizeTagList(c.KVMTags[vm]) {
+		current[t] = struct{}{}
+	}
+	changes := normalizeTagList(tags)
+	if len(changes) == 0 {
+		return Cluster{}, errors.New("at least one non-empty tag is required")
+	}
+	if add {
+		for _, t := range changes {
+			current[t] = struct{}{}
+		}
+	} else {
+		for _, t := range changes {
+			delete(current, t)
+		}
+	}
+	out := make([]string, 0, len(current))
+	for t := range current {
+		out = append(out, t)
+	}
+	sort.Strings(out)
+	if len(out) == 0 {
+		delete(c.KVMTags, vm)
+	} else {
+		c.KVMTags[vm] = out
+	}
+	c.KVMTags = normalizeVMTagMap(c.KVMTags)
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("kvm.tags", c.Name, vm+"="+strings.Join(out, ","))
+	return c, nil
+}
+
+func (s *Service) ListKVMTags(selector, vm string) (map[string][]string, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string][]string, len(c.KVMTags))
+	for k, v := range c.KVMTags {
+		if vm != "" && !strings.EqualFold(strings.TrimSpace(vm), k) {
+			continue
+		}
+		out[k] = append([]string(nil), normalizeTagList(v)...)
+	}
+	return out, nil
+}
diff --git a/internal/cluster/transport.go b/internal/cluster/transport.go
new file mode 100644
index 0000000..2201dab
--- /dev/null
+++ b/internal/cluster/transport.go
@@ -0,0 +1,269 @@
+package cluster
+
+import (
+	"context"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// agentClient wraps an *http.Client pointed at the correct base URL for a
+// cluster's agent (direct or SSH-tunneled).
+type agentClient struct {
+	http   *http.Client
+	target string
+	close  func()
+}
+
+func (a *agentClient) Close() {
+	if a != nil && a.close != nil {
+		a.close()
+	}
+}
+
+// newAgentClient builds the right HTTP client for reaching a cluster's agent.
+//
+// For TransportDirect it returns a plain client talking to cluster.Host:port.
+//
+// For TransportIPFabric it reuses a cached ssh.Client from the service pool
+// and returns a client whose Transport routes every TCP connection through
+// ssh.Client.Dial to 127.0.0.1:port. The SSH connection stays pooled after
+// Close() — only the HTTP transport's idle conns are released.
+func (s *Service) newAgentClient(ctx context.Context, c Cluster, timeout time.Duration) (*agentClient, error) {
+	if c.Agent.Port == 0 {
+		return nil, errors.New("agent port is not set")
+	}
+	if timeout <= 0 {
+		timeout = 8 * time.Second
+	}
+
+	scheme := agentScheme(c)
+	tlsCfg, err := agentTLSConfig(c)
+	if err != nil {
+		return nil, err
+	}
+
+	switch normalizeTransport(c.Transport) {
+	case TransportIPFabric:
+		sshClient, err := s.acquireTunnelClient(ctx, c)
+		if err != nil {
+			return nil, err
+		}
+		tr := &http.Transport{
+			TLSClientConfig: tlsCfg,
+			DialContext: func(dctx context.Context, network, _ string) (net.Conn, error) {
+				addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(c.Agent.Port))
+				return sshDialWithContext(dctx, sshClient, network, addr)
+			},
+			DisableKeepAlives:     true,
+			IdleConnTimeout:       30 * time.Second,
+			ResponseHeaderTimeout: timeout,
+			ExpectContinueTimeout: 1 * time.Second,
+		}
+		httpClient := &http.Client{
+			Timeout:   timeout,
+			Transport: tr,
+		}
+		return &agentClient{
+			http:   httpClient,
+			target: scheme + "://127.0.0.1:" + strconv.Itoa(c.Agent.Port),
+			close: func() {
+				tr.CloseIdleConnections()
+			},
+		}, nil
+
+	default:
+		tr := &http.Transport{
+			TLSClientConfig:       tlsCfg,
+			ResponseHeaderTimeout: timeout,
+			ExpectContinueTimeout: 1 * time.Second,
+		}
+		httpClient := &http.Client{Timeout: timeout, Transport: tr}
+		return &agentClient{
+			http:   httpClient,
+			target: scheme + "://" + net.JoinHostPort(c.Host, strconv.Itoa(c.Agent.Port)),
+			close: func() {
+				tr.CloseIdleConnections()
+			},
+		}, nil
+	}
+}
+
+func agentScheme(c Cluster) string {
+	if c.Agent.TLSEnabled {
+		return "https"
+	}
+	return "http"
+}
+
+func agentTLSConfig(c Cluster) (*tls.Config, error) {
+	if !c.Agent.TLSEnabled {
+		return nil, nil
+	}
+	fp := strings.ToLower(strings.TrimSpace(c.Agent.TLSFingerprint))
+	if fp == "" {
+		return nil, errors.New("agent TLS is enabled but certificate fingerprint is missing")
+	}
+	fp = strings.ReplaceAll(fp, ":", "")
+	want, err := hex.DecodeString(fp)
+	if err != nil {
+		return nil, fmt.Errorf("invalid agent TLS fingerprint: %w", err)
+	}
+	return &tls.Config{
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: true, // verified via explicit fingerprint pinning below
+		VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
+			if len(rawCerts) == 0 {
+				return errors.New("agent TLS: peer certificate is missing")
+			}
+			sum := sha256.Sum256(rawCerts[0])
+			if len(want) != len(sum) {
+				return errors.New("agent TLS: fingerprint length mismatch")
+			}
+			if !hmacEqual(sum[:], want) {
+				return errors.New("agent TLS: fingerprint mismatch")
+			}
+			return nil
+		},
+	}, nil
+}
+
+func hmacEqual(a, b []byte) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	var v byte
+	for i := 0; i < len(a); i++ {
+		v |= a[i] ^ b[i]
+	}
+	return v == 0
+}
+
+// acquireTunnelClient returns a pooled ssh.Client for the cluster, creating
+// one if necessary. Credential changes invalidate the cached entry via the
+// fingerprint field. Dead clients are evicted lazily: when a DialContext
+// through a stale client fails, the caller invokes CloseTunnelClient and the
+// next acquire re-dials.
+func (s *Service) acquireTunnelClient(ctx context.Context, c Cluster) (*ssh.Client, error) {
+	fp := credentialFingerprint(c)
+
+	s.tunnelMu.Lock()
+	entry, ok := s.tunnelPool[c.ID]
+	if ok && entry.fp != fp {
+		_ = entry.client.Close()
+		delete(s.tunnelPool, c.ID)
+		entry = nil
+		ok = false
+	}
+	if ok {
+		s.tunnelMu.Unlock()
+		return entry.client, nil
+	}
+	s.tunnelMu.Unlock()
+
+	client, err := s.dialSSH(ctx, c, "", "")
+	if err != nil {
+		return nil, err
+	}
+
+	s.tunnelMu.Lock()
+	if existing, ok := s.tunnelPool[c.ID]; ok && existing.fp == fp {
+		// Another goroutine won the race; drop ours.
+		s.tunnelMu.Unlock()
+		_ = client.Close()
+		return existing.client, nil
+	}
+	s.tunnelPool[c.ID] = &tunneledSSH{
+		client: client,
+		fp:     fp,
+	}
+	s.tunnelMu.Unlock()
+	return client, nil
+}
+
+// CloseTunnelClient drops a pooled SSH tunnel for a cluster. Safe to call if
+// no entry exists.
+func (s *Service) CloseTunnelClient(clusterID string) {
+	s.tunnelMu.Lock()
+	entry, ok := s.tunnelPool[clusterID]
+	if ok {
+		delete(s.tunnelPool, clusterID)
+	}
+	s.tunnelMu.Unlock()
+	if ok && entry != nil && entry.client != nil {
+		_ = entry.client.Close()
+	}
+}
+
+// CloseAllTunnelClients tears down every pooled SSH tunnel.
+func (s *Service) CloseAllTunnelClients() {
+	s.tunnelMu.Lock()
+	pool := s.tunnelPool
+	s.tunnelPool = make(map[string]*tunneledSSH)
+	s.tunnelMu.Unlock()
+	for _, e := range pool {
+		if e != nil && e.client != nil {
+			_ = e.client.Close()
+		}
+	}
+}
+
+// credentialFingerprint returns a short hash over the fields that affect how
+// we'd reconnect. If any of these change we must not reuse a cached client.
+func credentialFingerprint(c Cluster) string {
+	h := sha256.New()
+	h.Write([]byte(c.Host))
+	h.Write([]byte{'|'})
+	h.Write([]byte(strconv.Itoa(c.Port)))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.User))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.AuthMethod))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.Password))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.KeyPath))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.KeyPassphrase))
+	h.Write([]byte{'|'})
+	h.Write([]byte(c.KeyPassphraseFile))
+	return hex.EncodeToString(h.Sum(nil)[:8])
+}
+
+// sshDialWithContext wraps ssh.Client.Dial so it respects ctx cancellation.
+// ssh.Client has no context-aware dial, so we fall back to a watcher goroutine
+// that closes the connection if ctx fires before the dial returns.
+func sshDialWithContext(ctx context.Context, client *ssh.Client, network, addr string) (net.Conn, error) {
+	type result struct {
+		conn net.Conn
+		err  error
+	}
+	ch := make(chan result, 1)
+	go func() {
+		conn, err := client.Dial(network, addr)
+		ch <- result{conn: conn, err: err}
+	}()
+
+	select {
+	case <-ctx.Done():
+		go func() {
+			r := <-ch
+			if r.conn != nil {
+				_ = r.conn.Close()
+			}
+		}()
+		return nil, ctx.Err()
+	case r := <-ch:
+		return r.conn, r.err
+	}
+}
diff --git a/internal/cluster/usage.go b/internal/cluster/usage.go
new file mode 100644
index 0000000..d34600d
--- /dev/null
+++ b/internal/cluster/usage.go
@@ -0,0 +1,272 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"time"
+
+	"pxmon/internal/agent"
+	"pxmon/internal/history"
+)
+
+// UsageRange is a façade over history.RangeShortcut so callers outside this
+// package don't need to import both.
+type UsageRange = history.RangeShortcut
+
+// UsageSnapshot bundles everything the "usage" page shows for a cluster at a
+// single point in time.
+type UsageSnapshot struct {
+	ClusterID    string                    `json:"cluster_id"`
+	ClusterName  string                    `json:"cluster_name"`
+	Range        string                    `json:"range"`
+	GeneratedAt  time.Time                 `json:"generated_at"`
+	Live         agent.StatsResponse       `json:"live"`
+	Top          agent.TopResponse         `json:"top"`
+	DU           agent.DUResponse          `json:"du"`
+	NodeSeries   []history.NodeSamplePoint `json:"series,omitempty"`
+	P95TotalMbps float64                   `json:"p95_total_mbps"`
+	MaxTotalMbps float64                   `json:"max_total_mbps"`
+	AvgTotalMbps float64                   `json:"avg_total_mbps"`
+	TopIfaceName string                    `json:"top_iface_name,omitempty"`
+	TopIfaceMbps float64                   `json:"top_iface_mbps,omitempty"`
+	DUError      string                    `json:"du_error,omitempty"`
+	TopError     string                    `json:"top_error,omitempty"`
+	HistoryError string                    `json:"history_error,omitempty"`
+}
+
+// AgentTopProcesses calls /api/v1/top on the selected cluster's agent.
+func (s *Service) AgentTopProcesses(ctx context.Context, selector string, sampleWindow time.Duration, limit int) (agent.TopResponse, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return agent.TopResponse{}, err
+	}
+	if !cluster.Agent.Installed {
+		return agent.TopResponse{}, errors.New("agent is not installed on this cluster")
+	}
+
+	ac, err := s.newAgentClient(ctx, cluster, 15*time.Second)
+	if err != nil {
+		return agent.TopResponse{}, err
+	}
+	defer ac.Close()
+
+	url := ac.target + "/api/v1/top"
+	q := ""
+	if sampleWindow > 0 {
+		q += "sample_ms=" + strconv.FormatInt(sampleWindow.Milliseconds(), 10)
+	}
+	if limit > 0 {
+		if q != "" {
+			q += "&"
+		}
+		q += "limit=" + strconv.Itoa(limit)
+	}
+	if q != "" {
+		url += "?" + q
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return agent.TopResponse{}, err
+	}
+	applyAgentRequestAuth(req, cluster)
+
+	resp, err := ac.http.Do(req)
+	if err != nil {
+		if normalizeTransport(cluster.Transport) == TransportIPFabric {
+			s.CloseTunnelClient(cluster.ID)
+		}
+		return agent.TopResponse{}, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return agent.TopResponse{}, err
+	}
+	if resp.StatusCode >= 300 {
+		msg := string(body)
+		if len(msg) > 200 {
+			msg = msg[:200]
+		}
+		return agent.TopResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
+	}
+
+	var out agent.TopResponse
+	if err := json.Unmarshal(body, &out); err != nil {
+		return agent.TopResponse{}, err
+	}
+	return out, nil
+}
+
+// AgentDirSizes calls /api/v1/du on the selected cluster's agent.
+func (s *Service) AgentDirSizes(ctx context.Context, selector, path string, limit int, timeout time.Duration) (agent.DUResponse, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return agent.DUResponse{}, err
+	}
+	if !cluster.Agent.Installed {
+		return agent.DUResponse{}, errors.New("agent is not installed on this cluster")
+	}
+
+	httpTimeout := timeout + 10*time.Second
+	if httpTimeout < 20*time.Second {
+		httpTimeout = 20 * time.Second
+	}
+	ac, err := s.newAgentClient(ctx, cluster, httpTimeout)
+	if err != nil {
+		return agent.DUResponse{}, err
+	}
+	defer ac.Close()
+
+	url := ac.target + "/api/v1/du"
+	q := ""
+	if path != "" {
+		q += "path=" + path
+	}
+	if limit > 0 {
+		if q != "" {
+			q += "&"
+		}
+		q += "limit=" + strconv.Itoa(limit)
+	}
+	if timeout > 0 {
+		if q != "" {
+			q += "&"
+		}
+		q += "timeout_ms=" + strconv.FormatInt(timeout.Milliseconds(), 10)
+	}
+	if q != "" {
+		url += "?" + q
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return agent.DUResponse{}, err
+	}
+	applyAgentRequestAuth(req, cluster)
+
+	resp, err := ac.http.Do(req)
+	if err != nil {
+		if normalizeTransport(cluster.Transport) == TransportIPFabric {
+			s.CloseTunnelClient(cluster.ID)
+		}
+		return agent.DUResponse{}, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return agent.DUResponse{}, err
+	}
+	if resp.StatusCode >= 300 {
+		msg := string(body)
+		if len(msg) > 200 {
+			msg = msg[:200]
+		}
+		return agent.DUResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
+	}
+
+	var out agent.DUResponse
+	if err := json.Unmarshal(body, &out); err != nil {
+		return agent.DUResponse{}, err
+	}
+	return out, nil
+}
+
+// CollectUsageSnapshot gathers every data source needed by the usage page:
+// live stats, top processes, top folders, and historical aggregate for P95.
+// Errors in individual components are recorded on the snapshot instead of
+// aborting the whole call — the page degrades gracefully.
+func (s *Service) CollectUsageSnapshot(ctx context.Context, selector string, rng UsageRange, duPath string) (UsageSnapshot, error) {
+	cluster, err := s.Get(selector)
+	if err != nil {
+		return UsageSnapshot{}, err
+	}
+
+	snap := UsageSnapshot{
+		ClusterID:   cluster.ID,
+		ClusterName: cluster.Name,
+		Range:       string(rng),
+		GeneratedAt: time.Now().UTC(),
+	}
+
+	if live, err := s.AgentStatsTyped(ctx, selector); err != nil {
+		return snap, fmt.Errorf("live stats: %w", err)
+	} else {
+		snap.Live = live
+	}
+
+	if top, err := s.AgentTopProcesses(ctx, selector, 300*time.Millisecond, 15); err != nil {
+		snap.TopError = err.Error()
+	} else {
+		snap.Top = top
+	}
+
+	if duPath == "" {
+		duPath = "/"
+	}
+	if du, err := s.AgentDirSizes(ctx, selector, duPath, 12, 12*time.Second); err != nil {
+		snap.DUError = err.Error()
+	} else {
+		snap.DU = du
+	}
+
+	// Network history + P95 aggregation.
+	if s.networkStore == nil {
+		snap.HistoryError = "history store not configured"
+	} else {
+		since := history.RangeShortcut(rng).Since(time.Now())
+		if rng == "" || rng == history.RangeLive {
+			since = time.Now().Add(-5 * time.Minute)
+		}
+		snapshots, err := s.networkStore.Load(cluster.ID, since)
+		if err != nil {
+			snap.HistoryError = err.Error()
+		} else {
+			// Pick the physical uplink once and use it consistently for
+			// the series, P95, and the "top iface" display so they all
+			// describe the same thing.
+			primary := history.PrimaryInterface(snapshots)
+			series := history.AggregateNodeSeries(snapshots, primary)
+			snap.NodeSeries = series
+			snap.P95TotalMbps = history.PercentileMbps(series, 95)
+			if len(series) > 0 {
+				maxV := 0.0
+				sum := 0.0
+				for _, p := range series {
+					if p.TotalMbps > maxV {
+						maxV = p.TotalMbps
+					}
+					sum += p.TotalMbps
+				}
+				snap.MaxTotalMbps = maxV
+				snap.AvgTotalMbps = sum / float64(len(series))
+			}
+			snap.TopIfaceName = primary
+			snap.TopIfaceMbps = snap.AvgTotalMbps
+		}
+	}
+
+	return snap, nil
+}
+
+// RenderUsageChartPNG is a thin helper that reads a usage snapshot's series
+// and delegates to history.RenderNodeNetworkPNG.
+func RenderUsageChartPNG(snap UsageSnapshot, title string) ([]byte, error) {
+	if title == "" {
+		title = fmt.Sprintf("%s — node network usage (%s)", snap.ClusterName, snap.Range)
+	}
+	subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps",
+		snap.P95TotalMbps, snap.MaxTotalMbps, snap.AvgTotalMbps)
+	return history.RenderNodeNetworkPNG(snap.NodeSeries, history.ChartOptions{
+		Title:      title,
+		Subtitle:   subtitle,
+		Percentile: 95,
+	})
+}
diff --git a/internal/cluster/vm_alerts.go b/internal/cluster/vm_alerts.go
new file mode 100644
index 0000000..d4e8695
--- /dev/null
+++ b/internal/cluster/vm_alerts.go
@@ -0,0 +1,137 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type VMStateSummary struct {
+	Total        int       `json:"total"`
+	Running      int       `json:"running"`
+	ShutOff      int       `json:"shut_off"`
+	Paused       int       `json:"paused"`
+	Others       int       `json:"others"`
+	ShutOffNames []string  `json:"shut_off_names,omitempty"`
+	PausedNames  []string  `json:"paused_names,omitempty"`
+	OtherNames   []string  `json:"other_names,omitempty"`
+	Warnings     []string  `json:"warnings,omitempty"`
+	SampledAt    time.Time `json:"sampled_at"`
+}
+
+func (s *Service) GetVMAlertPolicy(selector string) (VMAlertPolicy, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return VMAlertPolicy{}, err
+	}
+	return ensureVMAlertPolicy(c.VMAlerts), nil
+}
+
+func (s *Service) SetVMAlertPolicy(selector string, p VMAlertPolicy) (Cluster, error) {
+	reg, err := s.store.Load()
+	if err != nil {
+		return Cluster{}, err
+	}
+	c, idx, err := findCluster(reg, selector)
+	if err != nil {
+		return Cluster{}, err
+	}
+	c.VMAlerts = ensureVMAlertPolicy(p)
+	c.UpdatedAt = s.now().UTC()
+	reg.Clusters[idx] = c
+	if err := s.store.Save(reg); err != nil {
+		return Cluster{}, err
+	}
+	_ = s.AppendChange("vm.alerts", c.Name, fmt.Sprintf("enabled=%t warn_on_shutoff=%t min_running=%d", c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning))
+	return c, nil
+}
+
+func (s *Service) CheckVMAlerts(ctx context.Context, selector string) (VMStateSummary, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return VMStateSummary{}, err
+	}
+	out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
+	if err != nil {
+		return VMStateSummary{}, err
+	}
+	states := parseVirshListStates(out)
+	summary := VMStateSummary{SampledAt: s.now().UTC(), Total: len(states)}
+	for name, st := range states {
+		n := strings.ToLower(strings.TrimSpace(st))
+		switch {
+		case n == "running":
+			summary.Running++
+		case n == "shut off":
+			summary.ShutOff++
+			summary.ShutOffNames = append(summary.ShutOffNames, name)
+		case n == "paused":
+			summary.Paused++
+			summary.PausedNames = append(summary.PausedNames, name)
+		default:
+			summary.Others++
+			summary.OtherNames = append(summary.OtherNames, name)
+		}
+	}
+	sort.Strings(summary.ShutOffNames)
+	sort.Strings(summary.PausedNames)
+	sort.Strings(summary.OtherNames)
+	p := ensureVMAlertPolicy(c.VMAlerts)
+	if p.Enabled {
+		if p.WarnOnShutoff && summary.ShutOff > 0 {
+			summary.Warnings = append(summary.Warnings, fmt.Sprintf(
+				"%d VM(s) are shut off: %s",
+				summary.ShutOff,
+				joinNamesLimit(summary.ShutOffNames, 12),
+			))
+		}
+		if summary.Running < p.MinRunning {
+			summary.Warnings = append(summary.Warnings, fmt.Sprintf("running VM count %d is below min_running=%d", summary.Running, p.MinRunning))
+		}
+	}
+	sort.Strings(summary.Warnings)
+	return summary, nil
+}
+
+func (s *Service) ListVMStates(ctx context.Context, selector string) (map[string]string, error) {
+	c, err := s.Get(selector)
+	if err != nil {
+		return nil, err
+	}
+	out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
+	if err != nil {
+		return nil, err
+	}
+	return parseVirshListStates(out), nil
+}
+
+func parseVirshListStates(raw string) map[string]string {
+	lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
+	out := make(map[string]string)
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "Id") || strings.HasPrefix(line, "-") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 3 {
+			continue
+		}
+		name := fields[1]
+		state := strings.Join(fields[2:], " ")
+		out[name] = state
+	}
+	return out
+}
+
+func joinNamesLimit(items []string, limit int) string {
+	if len(items) == 0 {
+		return "-"
+	}
+	if limit <= 0 || len(items) <= limit {
+		return strings.Join(items, ", ")
+	}
+	return strings.Join(items[:limit], ", ") + fmt.Sprintf(" (+%d more)", len(items)-limit)
+}
diff --git a/internal/history/availability_store.go b/internal/history/availability_store.go
new file mode 100644
index 0000000..d5f9229
--- /dev/null
+++ b/internal/history/availability_store.go
@@ -0,0 +1,92 @@
+package history
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type AvailabilitySnapshot struct {
+	Timestamp time.Time         `json:"timestamp"`
+	ClusterUp bool              `json:"cluster_up"`
+	VMStates  map[string]string `json:"vm_states,omitempty"`
+	Error     string            `json:"error,omitempty"`
+}
+
+type AvailabilityStore struct {
+	dir string
+}
+
+func NewAvailabilityStore(baseDir string) *AvailabilityStore {
+	if strings.TrimSpace(baseDir) == "" {
+		baseDir = "."
+	}
+	return &AvailabilityStore{dir: filepath.Join(baseDir, "history", "availability")}
+}
+
+func (s *AvailabilityStore) Path(clusterID string) string {
+	return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
+}
+
+func (s *AvailabilityStore) Append(clusterID string, snap AvailabilitySnapshot) error {
+	if strings.TrimSpace(clusterID) == "" {
+		return errors.New("empty cluster id")
+	}
+	if snap.Timestamp.IsZero() {
+		snap.Timestamp = time.Now().UTC()
+	}
+	if err := os.MkdirAll(s.dir, 0o700); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(s.Path(clusterID), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	b, err := json.Marshal(snap)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(append(b, '\n'))
+	return err
+}
+
+func (s *AvailabilityStore) Load(clusterID string, since time.Time) ([]AvailabilitySnapshot, error) {
+	if strings.TrimSpace(clusterID) == "" {
+		return nil, errors.New("empty cluster id")
+	}
+	f, err := os.Open(s.Path(clusterID))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return []AvailabilitySnapshot{}, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
+	out := make([]AvailabilitySnapshot, 0, 512)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var snap AvailabilitySnapshot
+		if err := json.Unmarshal([]byte(line), &snap); err != nil {
+			continue
+		}
+		if !since.IsZero() && snap.Timestamp.Before(since) {
+			continue
+		}
+		out = append(out, snap)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("scan availability: %w", err)
+	}
+	return out, nil
+}
diff --git a/internal/history/capacity_store.go b/internal/history/capacity_store.go
new file mode 100644
index 0000000..d456ba6
--- /dev/null
+++ b/internal/history/capacity_store.go
@@ -0,0 +1,102 @@
+package history
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type CapacityDiskPoint struct {
+	Mount      string `json:"mount"`
+	UsedBytes  uint64 `json:"used_bytes"`
+	TotalBytes uint64 `json:"total_bytes"`
+}
+
+type CapacitySnapshot struct {
+	Timestamp time.Time           `json:"timestamp"`
+	Disks     []CapacityDiskPoint `json:"disks,omitempty"`
+}
+
+type CapacityStore struct {
+	dir string
+}
+
+func NewCapacityStore(baseDir string) *CapacityStore {
+	if strings.TrimSpace(baseDir) == "" {
+		baseDir = "."
+	}
+	return &CapacityStore{dir: filepath.Join(baseDir, "history", "capacity")}
+}
+
+func (s *CapacityStore) Path(clusterID string) string {
+	return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
+}
+
+func (s *CapacityStore) Append(clusterID string, snap CapacitySnapshot) error {
+	if strings.TrimSpace(clusterID) == "" {
+		return errors.New("empty cluster id")
+	}
+	if snap.Timestamp.IsZero() {
+		snap.Timestamp = time.Now().UTC()
+	}
+	if len(snap.Disks) == 0 {
+		return nil
+	}
+	if err := os.MkdirAll(s.dir, 0o700); err != nil {
+		return err
+	}
+	f, err := os.OpenFile(s.Path(clusterID), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	b, err := json.Marshal(snap)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(append(b, '\n'))
+	return err
+}
+
+func (s *CapacityStore) Load(clusterID string, since time.Time) ([]CapacitySnapshot, error) {
+	if strings.TrimSpace(clusterID) == "" {
+		return nil, errors.New("empty cluster id")
+	}
+	f, err := os.Open(s.Path(clusterID))
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return []CapacitySnapshot{}, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
+	out := make([]CapacitySnapshot, 0, 512)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var snap CapacitySnapshot
+		if err := json.Unmarshal([]byte(line), &snap); err != nil {
+			continue
+		}
+		if !since.IsZero() && snap.Timestamp.Before(since) {
+			continue
+		}
+		if len(snap.Disks) == 0 {
+			continue
+		}
+		out = append(out, snap)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("scan capacity: %w", err)
+	}
+	return out, nil
+}
diff --git a/internal/history/chart.go b/internal/history/chart.go
new file mode 100644
index 0000000..26263f4
--- /dev/null
+++ b/internal/history/chart.go
@@ -0,0 +1,199 @@
+package history
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+
+	chart "github.com/wcharczuk/go-chart/v2"
+	"github.com/wcharczuk/go-chart/v2/drawing"
+)
+
+// ChartOptions controls rendering of the node network PNG.
+type ChartOptions struct {
+	Title      string
+	Subtitle   string
+	Width      int
+	Height     int
+	Percentile float64
+}
+
+// RenderNodeNetworkPNG returns a PNG of the node-wide Rx+Tx time series with
+// the requested percentile drawn as a horizontal annotation line.
+func RenderNodeNetworkPNG(points []NodeSamplePoint, opts ChartOptions) ([]byte, error) {
+	if opts.Width <= 0 {
+		opts.Width = 1280
+	}
+	if opts.Height <= 0 {
+		opts.Height = 640
+	}
+	if opts.Percentile <= 0 {
+		opts.Percentile = 95
+	}
+	if opts.Title == "" {
+		opts.Title = "Node network usage"
+	}
+
+	if len(points) < 2 {
+		return renderEmptyChart(opts, "insufficient history — need at least 2 samples")
+	}
+
+	xs := make([]time.Time, 0, len(points))
+	rxs := make([]float64, 0, len(points))
+	txs := make([]float64, 0, len(points))
+	totals := make([]float64, 0, len(points))
+	maxVal := 0.0
+	for _, p := range points {
+		xs = append(xs, p.Timestamp)
+		rxs = append(rxs, p.RxMbps)
+		txs = append(txs, p.TxMbps)
+		totals = append(totals, p.TotalMbps)
+		if p.TotalMbps > maxVal {
+			maxVal = p.TotalMbps
+		}
+	}
+
+	pct := PercentileMbps(points, opts.Percentile)
+	yMax := maxVal * 1.15
+	if pct*1.10 > yMax {
+		yMax = pct * 1.10
+	}
+	if yMax <= 0 {
+		yMax = 1
+	}
+
+	percentileSeries := chart.ContinuousSeries{
+		Name: fmt.Sprintf("P%.0f = %.1f Mbps", opts.Percentile, pct),
+		Style: chart.Style{
+			StrokeColor:     drawing.ColorFromHex("e74c3c"),
+			StrokeWidth:     2.0,
+			StrokeDashArray: []float64{6, 4},
+		},
+		XValues: []float64{chart.TimeToFloat64(xs[0]), chart.TimeToFloat64(xs[len(xs)-1])},
+		YValues: []float64{pct, pct},
+	}
+
+	graph := chart.Chart{
+		Title: opts.Title,
+		TitleStyle: chart.Style{
+			FontSize: 16,
+		},
+		Width:  opts.Width,
+		Height: opts.Height,
+		Background: chart.Style{
+			Padding: chart.Box{Top: 60, Left: 40, Right: 40, Bottom: 40},
+			FillColor: drawing.Color{
+				R: 0xf8, G: 0xf9, B: 0xfa, A: 0xff,
+			},
+		},
+		XAxis: chart.XAxis{
+			Style:          chart.Style{FontSize: 9},
+			ValueFormatter: chart.TimeValueFormatterWithFormat("15:04:05\n02 Jan"),
+		},
+		YAxis: chart.YAxis{
+			Name:  "Mbps",
+			Style: chart.Style{FontSize: 9},
+			Range: &chart.ContinuousRange{Min: 0, Max: yMax},
+			ValueFormatter: func(v any) string {
+				if f, ok := v.(float64); ok {
+					return formatMbps(f)
+				}
+				return ""
+			},
+		},
+		Series: []chart.Series{
+			chart.TimeSeries{
+				Name: "Total Rx+Tx",
+				Style: chart.Style{
+					StrokeColor: drawing.ColorFromHex("2d7dd2"),
+					StrokeWidth: 2.0,
+					FillColor:   drawing.ColorFromHex("2d7dd2").WithAlpha(50),
+				},
+				XValues: xs,
+				YValues: totals,
+			},
+			chart.TimeSeries{
+				Name: "Rx",
+				Style: chart.Style{
+					StrokeColor: drawing.ColorFromHex("3cb371"),
+					StrokeWidth: 1.5,
+				},
+				XValues: xs,
+				YValues: rxs,
+			},
+			chart.TimeSeries{
+				Name: "Tx",
+				Style: chart.Style{
+					StrokeColor: drawing.ColorFromHex("ffa500"),
+					StrokeWidth: 1.5,
+				},
+				XValues: xs,
+				YValues: txs,
+			},
+			percentileSeries,
+		},
+	}
+
+	if opts.Subtitle != "" {
+		graph.Elements = []chart.Renderable{subtitleRenderable(opts.Subtitle)}
+	}
+
+	buf := &bytes.Buffer{}
+	if err := graph.Render(chart.PNG, buf); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func subtitleRenderable(text string) chart.Renderable {
+	return func(r chart.Renderer, cb chart.Box, chartDefaults chart.Style) {
+		r.SetFont(chartDefaults.GetFont())
+		r.SetFontColor(drawing.Color{R: 90, G: 90, B: 90, A: 0xff})
+		r.SetFontSize(10)
+		r.Text(text, cb.Left+10, cb.Top+30)
+	}
+}
+
+func renderEmptyChart(opts ChartOptions, note string) ([]byte, error) {
+	graph := chart.Chart{
+		Title:  opts.Title,
+		Width:  opts.Width,
+		Height: opts.Height,
+		Background: chart.Style{
+			Padding: chart.Box{Top: 60, Left: 40, Right: 40, Bottom: 40},
+		},
+		Series: []chart.Series{
+			chart.ContinuousSeries{
+				XValues: []float64{0, 1},
+				YValues: []float64{0, 0},
+				Style: chart.Style{
+					StrokeColor: drawing.ColorTransparent,
+				},
+			},
+		},
+	}
+	graph.Elements = []chart.Renderable{
+		func(r chart.Renderer, cb chart.Box, cs chart.Style) {
+			r.SetFont(cs.GetFont())
+			r.SetFontColor(drawing.Color{R: 120, G: 120, B: 120, A: 0xff})
+			r.SetFontSize(14)
+			r.Text(note, cb.Left+20, cb.Top+cb.Height()/2)
+		},
+	}
+	buf := &bytes.Buffer{}
+	if err := graph.Render(chart.PNG, buf); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func formatMbps(v float64) string {
+	switch {
+	case v >= 1000:
+		return fmt.Sprintf("%.2f Gbps", v/1000)
+	case v >= 1:
+		return fmt.Sprintf("%.1f Mbps", v)
+	default:
+		return fmt.Sprintf("%.0f Kbps", v*1000)
+	}
+}
diff --git a/internal/history/network_store.go b/internal/history/network_store.go
new file mode 100644
index 0000000..4f9d9b3
--- /dev/null
+++ b/internal/history/network_store.go
@@ -0,0 +1,146 @@
+package history
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+// InterfaceSample is one per-interface network sample at timestamp.
+type InterfaceSample struct {
+	Interface string  `json:"interface"`
+	RxMbps    float64 `json:"rx_mbps"`
+	TxMbps    float64 `json:"tx_mbps"`
+	RxDrops   uint64  `json:"rx_drops"`
+	TxDrops   uint64  `json:"tx_drops"`
+}
+
+// NetworkSnapshot stores one full sample containing many interfaces.
+type NetworkSnapshot struct {
+	Timestamp  time.Time         `json:"timestamp"`
+	Interfaces []InterfaceSample `json:"interfaces"`
+}
+
+// NetworkStore appends and reads network snapshots (JSONL) per cluster.
+type NetworkStore struct {
+	dir string
+}
+
+func NewNetworkStore(baseDir string) *NetworkStore {
+	if strings.TrimSpace(baseDir) == "" {
+		baseDir = "."
+	}
+	return &NetworkStore{
+		dir: filepath.Join(baseDir, "history", "network"),
+	}
+}
+
+func (s *NetworkStore) Path(clusterID string) string {
+	return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
+}
+
+func (s *NetworkStore) Append(clusterID string, snapshot NetworkSnapshot) error {
+	if strings.TrimSpace(clusterID) == "" {
+		return errors.New("empty cluster id")
+	}
+	if snapshot.Timestamp.IsZero() {
+		snapshot.Timestamp = time.Now().UTC()
+	}
+	if len(snapshot.Interfaces) == 0 {
+		return nil
+	}
+
+	if err := os.MkdirAll(s.dir, 0o700); err != nil {
+		return fmt.Errorf("create history dir: %w", err)
+	}
+
+	path := s.Path(clusterID)
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
+	if err != nil {
+		return fmt.Errorf("open history file: %w", err)
+	}
+	defer f.Close()
+
+	line, err := json.Marshal(snapshot)
+	if err != nil {
+		return fmt.Errorf("marshal history snapshot: %w", err)
+	}
+	if _, err := f.Write(append(line, '\n')); err != nil {
+		return fmt.Errorf("append history snapshot: %w", err)
+	}
+	return nil
+}
+
+func (s *NetworkStore) Load(clusterID string, since time.Time) ([]NetworkSnapshot, error) {
+	if strings.TrimSpace(clusterID) == "" {
+		return nil, errors.New("empty cluster id")
+	}
+
+	path := s.Path(clusterID)
+	f, err := os.Open(path)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return []NetworkSnapshot{}, nil
+		}
+		return nil, fmt.Errorf("open history file: %w", err)
+	}
+	defer f.Close()
+
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 0, 1024*64), 1024*1024*8)
+
+	out := make([]NetworkSnapshot, 0, 256)
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		if line == "" {
+			continue
+		}
+		var snap NetworkSnapshot
+		if err := json.Unmarshal([]byte(line), &snap); err != nil {
+			continue
+		}
+		if !since.IsZero() && snap.Timestamp.Before(since) {
+			continue
+		}
+		if len(snap.Interfaces) == 0 {
+			continue
+		}
+		out = append(out, snap)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, fmt.Errorf("scan history file: %w", err)
+	}
+	return out, nil
+}
+
+func sanitizeClusterID(v string) string {
+	v = strings.TrimSpace(v)
+	if v == "" {
+		return "unknown"
+	}
+	var b strings.Builder
+	for _, r := range v {
+		switch {
+		case r >= 'a' && r <= 'z':
+			b.WriteRune(r)
+		case r >= 'A' && r <= 'Z':
+			b.WriteRune(r)
+		case r >= '0' && r <= '9':
+			b.WriteRune(r)
+		case r == '-' || r == '_':
+			b.WriteRune(r)
+		default:
+			b.WriteByte('_')
+		}
+	}
+	out := strings.Trim(b.String(), "_")
+	if out == "" {
+		return "unknown"
+	}
+	return out
+}
diff --git a/internal/history/percentile.go b/internal/history/percentile.go
new file mode 100644
index 0000000..d5fdd70
--- /dev/null
+++ b/internal/history/percentile.go
@@ -0,0 +1,242 @@
+package history
+
+import (
+	"math"
+	"sort"
+	"strings"
+	"time"
+)
+
+// NodeSamplePoint is one aggregated sample for the node's primary uplink
+// interface at a single point in time. TotalMbps uses max(Rx, Tx) which is
+// the same convention a provider uses for 95th-percentile transit billing.
+type NodeSamplePoint struct {
+	Timestamp time.Time
+	TotalMbps float64 // max(Rx, Tx) for billing P95
+	RxMbps    float64
+	TxMbps    float64
+	Interface string
+}
+
+// isVirtualIface returns true for interfaces that do not represent real
+// uplink traffic and should be excluded from P95/billing calculations.
+// Virtual interfaces (bridges, taps, veth pairs, docker, loopback) either
+// carry no real traffic or mirror the traffic that already flows through
+// the physical uplink — double-counting them inflates totals.
+func isVirtualIface(name string) bool {
+	n := strings.ToLower(name)
+	if n == "lo" || n == "" {
+		return true
+	}
+	prefixes := []string{
+		"lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr",
+		"cni", "flannel", "wg", "tun", "tailscale", "zt", "ipsec",
+		"kube", "cilium", "ovs", "podman", "dummy",
+	}
+	for _, p := range prefixes {
+		if strings.HasPrefix(n, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// PrimaryInterface picks the physical uplink interface with the highest
+// average max(Rx, Tx) over the given snapshots. Virtual interfaces are
+// ignored. Returns empty string when no physical candidate exists.
+func PrimaryInterface(snapshots []NetworkSnapshot) string {
+	type acc struct {
+		sum   float64
+		count int
+	}
+	totals := make(map[string]*acc, 16)
+	for _, snap := range snapshots {
+		for _, iface := range snap.Interfaces {
+			if isVirtualIface(iface.Interface) {
+				continue
+			}
+			v := math.Max(iface.RxMbps, iface.TxMbps)
+			a, ok := totals[iface.Interface]
+			if !ok {
+				a = &acc{}
+				totals[iface.Interface] = a
+			}
+			a.sum += v
+			a.count++
+		}
+	}
+	best := ""
+	bestAvg := -1.0
+	for name, a := range totals {
+		if a.count == 0 {
+			continue
+		}
+		avg := a.sum / float64(a.count)
+		if avg > bestAvg {
+			bestAvg = avg
+			best = name
+		}
+	}
+	return best
+}
+
+// AggregateNodeSeries builds a time series for a single interface. If
+// ifaceName is empty, PrimaryInterface is used to auto-select the physical
+// uplink. TotalMbps uses max(Rx, Tx), matching provider billing convention.
+func AggregateNodeSeries(snapshots []NetworkSnapshot, ifaceName string) []NodeSamplePoint {
+	if ifaceName == "" {
+		ifaceName = PrimaryInterface(snapshots)
+	}
+	out := make([]NodeSamplePoint, 0, len(snapshots))
+	for _, snap := range snapshots {
+		for _, iface := range snap.Interfaces {
+			if iface.Interface != ifaceName {
+				continue
+			}
+			rx := iface.RxMbps
+			tx := iface.TxMbps
+			out = append(out, NodeSamplePoint{
+				Timestamp: snap.Timestamp,
+				Interface: ifaceName,
+				RxMbps:    rx,
+				TxMbps:    tx,
+				TotalMbps: math.Max(rx, tx),
+			})
+			break
+		}
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Timestamp.Before(out[j].Timestamp)
+	})
+	return out
+}
+
+// PercentileMbps returns the given percentile (0..100) of the TotalMbps
+// field across the series using nearest-rank (inclusive) computation.
+// Returns 0 if the series is empty.
+func PercentileMbps(points []NodeSamplePoint, percentile float64) float64 {
+	if len(points) == 0 {
+		return 0
+	}
+	if percentile < 0 {
+		percentile = 0
+	}
+	if percentile > 100 {
+		percentile = 100
+	}
+	values := make([]float64, 0, len(points))
+	for _, p := range points {
+		values = append(values, p.TotalMbps)
+	}
+	sort.Float64s(values)
+	if len(values) == 1 {
+		return values[0]
+	}
+	rank := (percentile / 100.0) * float64(len(values)-1)
+	lo := int(math.Floor(rank))
+	hi := int(math.Ceil(rank))
+	if lo == hi {
+		return values[lo]
+	}
+	frac := rank - float64(lo)
+	return values[lo]*(1-frac) + values[hi]*frac
+}
+
+// TopInterfaceByTraffic returns the physical interface with the highest
+// average max(Rx, Tx) across the sampled period, together with that
+// average throughput in Mbps. Virtual interfaces are ignored.
+func TopInterfaceByTraffic(snapshots []NetworkSnapshot) (string, float64) {
+	type acc struct {
+		sum   float64
+		count int
+	}
+	totals := make(map[string]*acc, 16)
+	for _, snap := range snapshots {
+		for _, iface := range snap.Interfaces {
+			if isVirtualIface(iface.Interface) {
+				continue
+			}
+			v := math.Max(iface.RxMbps, iface.TxMbps)
+			a, ok := totals[iface.Interface]
+			if !ok {
+				a = &acc{}
+				totals[iface.Interface] = a
+			}
+			a.sum += v
+			a.count++
+		}
+	}
+	name := ""
+	bestAvg := 0.0
+	for k, a := range totals {
+		if a.count == 0 {
+			continue
+		}
+		avg := a.sum / float64(a.count)
+		if avg > bestAvg {
+			bestAvg = avg
+			name = k
+		}
+	}
+	return name, bestAvg
+}
+
+// RangeShortcut is a common time-window selector.
+type RangeShortcut string
+
+const (
+	RangeLive  RangeShortcut = "live"
+	RangeHour  RangeShortcut = "1h"
+	RangeDay   RangeShortcut = "1d"
+	RangeMonth RangeShortcut = "1mo"
+	RangeAll   RangeShortcut = "all"
+)
+
+// Since returns an absolute start time for the given range shortcut,
+// relative to now. RangeAll and RangeLive return zero (no lower bound).
+func (r RangeShortcut) Since(now time.Time) time.Time {
+	switch r {
+	case RangeHour:
+		return now.Add(-time.Hour)
+	case RangeDay:
+		return now.Add(-24 * time.Hour)
+	case RangeMonth:
+		return now.Add(-30 * 24 * time.Hour)
+	default:
+		return time.Time{}
+	}
+}
+
+// ParseRangeShortcut accepts user-provided range strings.
+func ParseRangeShortcut(raw string) (RangeShortcut, bool) {
+	switch raw {
+	case "live", "now":
+		return RangeLive, true
+	case "1h", "hour":
+		return RangeHour, true
+	case "1d", "day", "24h":
+		return RangeDay, true
+	case "1mo", "30d", "month":
+		return RangeMonth, true
+	case "all", "":
+		return RangeAll, true
+	}
+	return "", false
+}
+
+// Label returns a human-friendly label for the range.
+func (r RangeShortcut) Label() string {
+	switch r {
+	case RangeLive:
+		return "live"
+	case RangeHour:
+		return "last 1h"
+	case RangeDay:
+		return "last 24h"
+	case RangeMonth:
+		return "last 30d"
+	case RangeAll:
+		return "all time"
+	}
+	return string(r)
+}