endssh/pxmon
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

chore: publish pxmon v0.2.0

Browse Source
This commit is contained in:
endssh
2026-06-16 21:52:10 +04:00
commit 6b703db02b
69 changed files with 25886 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/agent/server.go
+763
View File
@@ -0,0 +1,763 @@
package agent
import (
"bufio"
"context"
"crypto/hmac"
"crypto/sha256"
"crypto/subtle"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math"
"net/http"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
)
// Version is set via ldflags at build time.
var Version = "v0.2.0"
// Config controls agent runtime.
type Config struct {
ListenAddr string `json:"listen_addr"`
Token string `json:"token"`
RequestSecret string `json:"request_secret,omitempty"`
TLSEnabled bool `json:"tls_enabled,omitempty"`
TLSCertPath string `json:"tls_cert_path,omitempty"`
TLSKeyPath string `json:"tls_key_path,omitempty"`
}
var (
cpuSampleMu sync.Mutex
prevCPUTotal uint64
prevCPUIdle uint64
prevCPUSet bool
)
const signedRequestMaxSkewSeconds = 300
type nonceState struct {
mu sync.Mutex
seen map[string]int64
}
// StatsResponse contains basic node metrics.
type StatsResponse struct {
Timestamp time.Time `json:"timestamp"`
Host HostStats `json:"host"`
CPU CPUStats `json:"cpu"`
Memory MemoryStats `json:"memory"`
Disk []DiskStats `json:"disk"`
Network []NetworkStat `json:"network"`
}
// HostStats contains host-level facts.
type HostStats struct {
Hostname string `json:"hostname"`
OS string `json:"os"`
Arch string `json:"arch"`
UptimeSeconds float64 `json:"uptime_seconds"`
}
// CPUStats contains CPU/load values.
type CPUStats struct {
LogicalCores int `json:"logical_cores"`
UsagePercent float64 `json:"usage_percent"`
Load1 float64 `json:"load_1"`
Load5 float64 `json:"load_5"`
Load15 float64 `json:"load_15"`
}
// MemoryStats contains RAM usage values in bytes.
type MemoryStats struct {
TotalBytes uint64 `json:"total_bytes"`
AvailableBytes uint64 `json:"available_bytes"`
UsedBytes uint64 `json:"used_bytes"`
UsedPercent float64 `json:"used_percent"`
SwapTotalBytes uint64 `json:"swap_total_bytes"`
SwapFreeBytes uint64 `json:"swap_free_bytes"`
SwapUsedBytes uint64 `json:"swap_used_bytes"`
SwapUsedPct float64 `json:"swap_used_percent"`
}
// DiskStats contains one mount usage entry.
type DiskStats struct {
Source string `json:"source"`
MountPoint string `json:"mount_point"`
FSType string `json:"fs_type"`
Device string `json:"device"`
TotalBytes uint64 `json:"total_bytes"`
FreeBytes uint64 `json:"free_bytes"`
UsedBytes uint64 `json:"used_bytes"`
UsedPercent float64 `json:"used_percent"`
ReadOnly bool `json:"read_only"`
DeviceState string `json:"device_state,omitempty"`
ErrorsCount uint64 `json:"errors_count"`
Health string `json:"health"`
Warnings []string `json:"warnings,omitempty"`
}
// NetworkStat contains one interface counters snapshot.
type NetworkStat struct {
Interface string `json:"interface"`
RxBytes uint64 `json:"rx_bytes"`
TxBytes uint64 `json:"tx_bytes"`
RxPackets uint64 `json:"rx_packets"`
TxPackets uint64 `json:"tx_packets"`
RxDrops uint64 `json:"rx_drops"`
TxDrops uint64 `json:"tx_drops"`
}
// LoadConfig reads JSON config from disk.
func LoadConfig(path string) (Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return Config{}, err
}
var cfg Config
if err := json.Unmarshal(data, &cfg); err != nil {
return Config{}, err
}
cfg.ListenAddr = strings.TrimSpace(cfg.ListenAddr)
cfg.Token = strings.TrimSpace(cfg.Token)
cfg.RequestSecret = strings.TrimSpace(cfg.RequestSecret)
cfg.TLSCertPath = strings.TrimSpace(cfg.TLSCertPath)
cfg.TLSKeyPath = strings.TrimSpace(cfg.TLSKeyPath)
if cfg.ListenAddr == "" {
cfg.ListenAddr = "0.0.0.0:19090"
}
if cfg.Token == "" {
return Config{}, errors.New("token is required")
}
if cfg.TLSEnabled {
if cfg.TLSCertPath == "" || cfg.TLSKeyPath == "" {
return Config{}, errors.New("tls_enabled=true requires tls_cert_path and tls_key_path")
}
}
return cfg, nil
}
// Run starts the HTTP API server.
func Run(cfg Config) error {
mux := http.NewServeMux()
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok\n"))
})
mux.Handle("/api/v1/ping", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
writeJSON(w, http.StatusOK, map[string]any{
"status": "ok",
"version": Version,
"time": time.Now().UTC(),
})
})))
mux.Handle("/api/v1/stats", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
stats, err := CollectStats()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, stats)
})))
mux.Handle("/api/v1/top", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
q := r.URL.Query()
limit, _ := strconv.Atoi(q.Get("limit"))
sampleMs, _ := strconv.Atoi(q.Get("sample_ms"))
window := time.Duration(sampleMs) * time.Millisecond
top, err := CollectTopProcesses(window, limit)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, top)
})))
mux.Handle("/api/v1/du", authMiddleware(cfg.Token, cfg.RequestSecret, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
return
}
q := r.URL.Query()
root := q.Get("path")
if strings.TrimSpace(root) == "" {
root = "/"
}
limit, _ := strconv.Atoi(q.Get("limit"))
timeoutMs, _ := strconv.Atoi(q.Get("timeout_ms"))
if timeoutMs <= 0 || timeoutMs > 45000 {
timeoutMs = 15000
}
ctx, cancel := context.WithTimeout(r.Context(), time.Duration(timeoutMs)*time.Millisecond)
defer cancel()
resp, err := CollectDirSizes(ctx, root, limit)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, resp)
})))
srv := &http.Server{
Addr: cfg.ListenAddr,
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
}
if cfg.TLSEnabled {
return srv.ListenAndServeTLS(cfg.TLSCertPath, cfg.TLSKeyPath)
}
return srv.ListenAndServe()
}
// CollectStats gathers a metrics snapshot.
func CollectStats() (StatsResponse, error) {
hostname, _ := os.Hostname()
uptime := readUptimeSeconds()
load1, load5, load15 := readLoadAvg()
usage := readCPUUsagePercent()
mem := readMemory()
disk := readDiskStats()
netStats := readNetworkStats()
stats := StatsResponse{
Timestamp: time.Now().UTC(),
Host: HostStats{
Hostname: hostname,
OS: runtime.GOOS,
Arch: runtime.GOARCH,
UptimeSeconds: uptime,
},
CPU: CPUStats{
LogicalCores: runtime.NumCPU(),
UsagePercent: usage,
Load1: round2(load1),
Load5: round2(load5),
Load15: round2(load15),
},
Memory: mem,
Disk: disk,
Network: netStats,
}
return stats, nil
}
func authMiddleware(token, requestSecret string, next http.Handler) http.Handler {
ns := &nonceState{seen: map[string]int64{}}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
auth := strings.TrimSpace(r.Header.Get("Authorization"))
xToken := strings.TrimSpace(r.Header.Get("X-Agent-Token"))
ok := false
if strings.HasPrefix(strings.ToLower(auth), "bearer ") {
provided := strings.TrimSpace(auth[7:])
if secureTokenEqual(provided, token) {
ok = true
}
}
if secureTokenEqual(xToken, token) {
ok = true
}
if !ok {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
if !verifySignedRequest(r, requestSecret, ns) {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
next.ServeHTTP(w, r)
})
}
func secureTokenEqual(provided, expected string) bool {
if len(provided) == 0 || len(expected) == 0 {
return false
}
return subtle.ConstantTimeCompare([]byte(provided), []byte(expected)) == 1
}
func verifySignedRequest(r *http.Request, secret string, ns *nonceState) bool {
if strings.TrimSpace(secret) == "" {
return true
}
tsRaw := strings.TrimSpace(r.Header.Get("X-Observer-Ts"))
nonce := strings.TrimSpace(r.Header.Get("X-Observer-Nonce"))
sigRaw := strings.TrimSpace(strings.ToLower(r.Header.Get("X-Observer-Signature")))
if tsRaw == "" || nonce == "" || sigRaw == "" {
return false
}
ts, err := strconv.ParseInt(tsRaw, 10, 64)
if err != nil {
return false
}
now := time.Now().UTC().Unix()
if ts < now-signedRequestMaxSkewSeconds || ts > now+signedRequestMaxSkewSeconds {
return false
}
payload := r.Method + "\n" + r.URL.RequestURI() + "\n" + tsRaw + "\n" + nonce
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(payload))
wantHex := hex.EncodeToString(mac.Sum(nil))
if subtle.ConstantTimeCompare([]byte(sigRaw), []byte(strings.ToLower(wantHex))) != 1 {
return false
}
key := tsRaw + ":" + nonce
ns.lock()
defer ns.unlock()
if expiry, ok := ns.get(key); ok && expiry >= now {
return false
}
ns.set(key, now+signedRequestMaxSkewSeconds+60)
ns.prune(now)
return true
}
func (n *nonceState) lock() { n.mu.Lock() }
func (n *nonceState) unlock() { n.mu.Unlock() }
func (n *nonceState) get(k string) (int64, bool) {
v, ok := n.seen[k]
return v, ok
}
func (n *nonceState) set(k string, exp int64) {
n.seen[k] = exp
}
func (n *nonceState) prune(now int64) {
if len(n.seen) < 4096 {
return
}
for k, exp := range n.seen {
if exp < now {
delete(n.seen, k)
}
}
}
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
_ = enc.Encode(v)
}
func readUptimeSeconds() float64 {
if runtime.GOOS == "linux" {
data, err := os.ReadFile("/proc/uptime")
if err == nil {
parts := strings.Fields(string(data))
if len(parts) > 0 {
if v, err := strconv.ParseFloat(parts[0], 64); err == nil {
return v
}
}
}
}
return 0
}
func readLoadAvg() (float64, float64, float64) {
if runtime.GOOS != "linux" {
return 0, 0, 0
}
data, err := os.ReadFile("/proc/loadavg")
if err != nil {
return 0, 0, 0
}
parts := strings.Fields(string(data))
if len(parts) < 3 {
return 0, 0, 0
}
l1, _ := strconv.ParseFloat(parts[0], 64)
l5, _ := strconv.ParseFloat(parts[1], 64)
l15, _ := strconv.ParseFloat(parts[2], 64)
return l1, l5, l15
}
func readCPUUsagePercent() float64 {
if runtime.GOOS != "linux" {
return 0
}
total, idle, err := readCPUCounters()
if err != nil {
return 0
}
cpuSampleMu.Lock()
defer cpuSampleMu.Unlock()
if !prevCPUSet {
prevCPUTotal = total
prevCPUIdle = idle
prevCPUSet = true
return 0
}
var usage float64
if total > prevCPUTotal {
dTotal := total - prevCPUTotal
dIdle := uint64(0)
if idle > prevCPUIdle {
dIdle = idle - prevCPUIdle
}
if dTotal > 0 {
if dIdle > dTotal {
dIdle = dTotal
}
usage = (float64(dTotal-dIdle) / float64(dTotal)) * 100
}
}
prevCPUTotal = total
prevCPUIdle = idle
if usage < 0 {
usage = 0
}
if usage > 100 {
usage = 100
}
return round2(usage)
}
func readCPUCounters() (uint64, uint64, error) {
data, err := os.ReadFile("/proc/stat")
if err != nil {
return 0, 0, err
}
lines := strings.Split(string(data), "\n")
if len(lines) == 0 {
return 0, 0, errors.New("empty /proc/stat")
}
fields := strings.Fields(lines[0])
if len(fields) < 5 || fields[0] != "cpu" {
return 0, 0, errors.New("invalid cpu line in /proc/stat")
}
var values []uint64
for i := 1; i < len(fields); i++ {
v, convErr := strconv.ParseUint(fields[i], 10, 64)
if convErr != nil {
return 0, 0, convErr
}
values = append(values, v)
}
total := uint64(0)
for _, v := range values {
total += v
}
idle := values[3]
if len(values) > 4 {
idle += values[4] // iowait
}
return total, idle, nil
}
func readMemory() MemoryStats {
if runtime.GOOS != "linux" {
return MemoryStats{}
}
f, err := os.Open("/proc/meminfo")
if err != nil {
return MemoryStats{}
}
defer f.Close()
values := map[string]uint64{}
s := bufio.NewScanner(f)
for s.Scan() {
line := s.Text()
parts := strings.Split(line, ":")
if len(parts) != 2 {
continue
}
key := strings.TrimSpace(parts[0])
fields := strings.Fields(strings.TrimSpace(parts[1]))
if len(fields) == 0 {
continue
}
v, err := strconv.ParseUint(fields[0], 10, 64)
if err != nil {
continue
}
values[key] = v * 1024 // kB -> bytes
}
total := values["MemTotal"]
available := values["MemAvailable"]
if available == 0 {
available = values["MemFree"] + values["Buffers"] + values["Cached"]
}
used := uint64(0)
if total > available {
used = total - available
}
usedPercent := 0.0
if total > 0 {
usedPercent = (float64(used) / float64(total)) * 100
}
swapTotal := values["SwapTotal"]
swapFree := values["SwapFree"]
swapUsed := uint64(0)
if swapTotal > swapFree {
swapUsed = swapTotal - swapFree
}
swapUsedPct := 0.0
if swapTotal > 0 {
swapUsedPct = (float64(swapUsed) / float64(swapTotal)) * 100
}
return MemoryStats{
TotalBytes: total,
AvailableBytes: available,
UsedBytes: used,
UsedPercent: round2(usedPercent),
SwapTotalBytes: swapTotal,
SwapFreeBytes: swapFree,
SwapUsedBytes: swapUsed,
SwapUsedPct: round2(swapUsedPct),
}
}
type mountInfo struct {
Source string
MountPoint string
FSType string
}
func readDiskStats() []DiskStats {
mounts := discoverMounts()
seen := make(map[string]struct{}, len(mounts))
out := make([]DiskStats, 0, len(mounts))
for _, m := range mounts {
if _, ok := seen[m.MountPoint]; ok {
continue
}
seen[m.MountPoint] = struct{}{}
total, free, err := statfsUsage(m.MountPoint)
if err != nil {
continue
}
used := uint64(0)
if total > free {
used = total - free
}
usedPercent := 0.0
if total > 0 {
usedPercent = (float64(used) / float64(total)) * 100
}
device, readOnly, state, errorsCount, health, warnings := readDiskHealth(m.Source, m.FSType)
out = append(out, DiskStats{
Source: m.Source,
MountPoint: m.MountPoint,
FSType: m.FSType,
Device: device,
TotalBytes: total,
FreeBytes: free,
UsedBytes: used,
UsedPercent: round2(usedPercent),
ReadOnly: readOnly,
DeviceState: state,
ErrorsCount: errorsCount,
Health: health,
Warnings: warnings,
})
}
sort.Slice(out, func(i, j int) bool { return out[i].MountPoint < out[j].MountPoint })
return out
}
func discoverMounts() []mountInfo {
if runtime.GOOS != "linux" {
return []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}}
}
f, err := os.Open("/proc/mounts")
if err != nil {
return []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}}
}
defer f.Close()
skipFS := map[string]struct{}{
"proc": {}, "sysfs": {}, "tmpfs": {}, "devtmpfs": {}, "devpts": {},
"overlay": {}, "squashfs": {}, "cgroup": {}, "cgroup2": {}, "autofs": {},
"securityfs": {}, "pstore": {}, "debugfs": {}, "tracefs": {}, "fusectl": {},
}
out := []mountInfo{}
s := bufio.NewScanner(f)
for s.Scan() {
line := s.Text()
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
fsType := fields[2]
if _, skip := skipFS[fsType]; skip {
continue
}
out = append(out, mountInfo{
Source: fields[0],
MountPoint: fields[1],
FSType: fsType,
})
}
if len(out) == 0 {
out = []mountInfo{{Source: "/", MountPoint: "/", FSType: "unknown"}}
}
return out
}
func readDiskHealth(source, fsType string) (device string, readOnly bool, state string, errorsCount uint64, health string, warnings []string) {
health = "ok"
if !strings.HasPrefix(source, "/dev/") {
return "", false, "", 0, health, nil
}
device = filepath.Base(source)
if strings.HasPrefix(device, "mapper/") {
device = strings.TrimPrefix(device, "mapper/")
}
device = strings.TrimPrefix(device, "/")
roPath := filepath.Join("/sys/class/block", device, "ro")
if v, err := os.ReadFile(roPath); err == nil {
readOnly = strings.TrimSpace(string(v)) == "1"
if readOnly {
health = "warning"
warnings = append(warnings, "device is read-only")
}
}
statePath := filepath.Join("/sys/class/block", device, "device", "state")
if v, err := os.ReadFile(statePath); err == nil {
state = strings.TrimSpace(string(v))
if state != "" && state != "running" {
if health == "ok" {
health = "warning"
}
warnings = append(warnings, "device state is "+state)
}
}
if fsType == "ext4" {
ext4ErrPath := filepath.Join("/sys/fs/ext4", device, "errors_count")
if v, err := os.ReadFile(ext4ErrPath); err == nil {
if parsed, convErr := strconv.ParseUint(strings.TrimSpace(string(v)), 10, 64); convErr == nil {
errorsCount = parsed
if errorsCount > 0 {
health = "critical"
warnings = append(warnings, fmt.Sprintf("ext4 errors_count=%d", errorsCount))
}
}
}
}
return device, readOnly, state, errorsCount, health, warnings
}
func readNetworkStats() []NetworkStat {
if runtime.GOOS != "linux" {
return []NetworkStat{}
}
f, err := os.Open("/proc/net/dev")
if err != nil {
return []NetworkStat{}
}
defer f.Close()
out := []NetworkStat{}
s := bufio.NewScanner(f)
lineNo := 0
for s.Scan() {
lineNo++
if lineNo <= 2 {
continue
}
line := strings.TrimSpace(s.Text())
parts := strings.Split(line, ":")
if len(parts) != 2 {
continue
}
iface := strings.TrimSpace(parts[0])
fields := strings.Fields(parts[1])
if len(fields) < 16 {
continue
}
rx, err1 := strconv.ParseUint(fields[0], 10, 64)
rxPackets, errP1 := strconv.ParseUint(fields[1], 10, 64)
rxDrops, errD1 := strconv.ParseUint(fields[3], 10, 64)
tx, err2 := strconv.ParseUint(fields[8], 10, 64)
txPackets, errP2 := strconv.ParseUint(fields[9], 10, 64)
txDrops, errD2 := strconv.ParseUint(fields[11], 10, 64)
if err1 != nil || err2 != nil || errP1 != nil || errP2 != nil || errD1 != nil || errD2 != nil {
continue
}
out = append(out, NetworkStat{
Interface: iface,
RxBytes: rx,
TxBytes: tx,
RxPackets: rxPackets,
TxPackets: txPackets,
RxDrops: rxDrops,
TxDrops: txDrops,
})
}
sort.Slice(out, func(i, j int) bool { return out[i].Interface < out[j].Interface })
return out
}
func round2(v float64) float64 {
return math.Round(v*100) / 100
}
// DefaultConfigPath returns default location used by agent process.
func DefaultConfigPath() string {
return filepath.Join(".", "agent.json")
}
internal/agent/statfs_unix.go
+13
View File
@@ -0,0 +1,13 @@
//go:build !windows
package agent
import "syscall"
func statfsUsage(path string) (total, free uint64, err error) {
var fs syscall.Statfs_t
if err := syscall.Statfs(path, &fs); err != nil {
return 0, 0, err
}
return fs.Blocks * uint64(fs.Bsize), fs.Bavail * uint64(fs.Bsize), nil
}
internal/agent/statfs_windows.go
+10
View File
@@ -0,0 +1,10 @@
//go:build windows
package agent
// statfsUsage is a compatibility fallback for Windows builds where
// syscall.Statfs is unavailable. We return zero-sized stats instead of
// failing compilation; disk health/usage can be extended with native APIs.
func statfsUsage(path string) (total, free uint64, err error) {
return 0, 0, nil
}
internal/agent/top.go
+431
View File
@@ -0,0 +1,431 @@
package agent
import (
"bufio"
"context"
"errors"
"io/fs"
"os"
"os/user"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
)
// ProcessStat is one process summary for /api/v1/top.
type ProcessStat struct {
PID int `json:"pid"`
User string `json:"user"`
Command string `json:"command"`
CPUPercent float64 `json:"cpu_percent"`
RSSBytes uint64 `json:"rss_bytes"`
VSZBytes uint64 `json:"vsz_bytes"`
IOReadTot uint64 `json:"io_read_bytes"`
IOWriteTot uint64 `json:"io_write_bytes"`
}
// TopResponse is the payload returned by /api/v1/top.
type TopResponse struct {
Timestamp time.Time `json:"timestamp"`
SampleMs int64 `json:"sample_ms"`
TotalProcs int `json:"total_procs"`
TopByCPU []ProcessStat `json:"top_by_cpu"`
TopByMemory []ProcessStat `json:"top_by_memory"`
TopByIO []ProcessStat `json:"top_by_io"`
}
// CollectTopProcesses takes two samples of /proc/[pid]/stat separated by
// sampleWindow to compute CPU%, then returns the top N processes by CPU, by
// RSS, and by IO total. Linux-only; other OSes return an empty response.
func CollectTopProcesses(sampleWindow time.Duration, limit int) (TopResponse, error) {
if runtime.GOOS != "linux" {
return TopResponse{Timestamp: time.Now().UTC()}, nil
}
if sampleWindow <= 0 {
sampleWindow = 250 * time.Millisecond
}
if sampleWindow > 2*time.Second {
sampleWindow = 2 * time.Second
}
if limit <= 0 || limit > 200 {
limit = 20
}
hz := clockTicksPerSecond()
pageSize := uint64(os.Getpagesize())
first, err := snapshotProcesses()
if err != nil {
return TopResponse{}, err
}
time.Sleep(sampleWindow)
second, err := snapshotProcesses()
if err != nil {
return TopResponse{}, err
}
usernameCache := newUsernameCache()
elapsedTicks := float64(sampleWindow.Seconds()) * hz
if elapsedTicks <= 0 {
elapsedTicks = 1
}
merged := make([]ProcessStat, 0, len(second))
for pid, s2 := range second {
s1, ok := first[pid]
cpu := 0.0
if ok {
dTicks := float64((s2.utime + s2.stime) - (s1.utime + s1.stime))
if dTicks > 0 {
cpu = (dTicks / elapsedTicks) * 100.0
}
}
if cpu < 0 {
cpu = 0
}
merged = append(merged, ProcessStat{
PID: pid,
User: usernameCache.lookup(s2.uid),
Command: s2.command,
CPUPercent: round2(cpu),
RSSBytes: s2.rssPages * pageSize,
VSZBytes: s2.vsize,
IOReadTot: s2.ioRead,
IOWriteTot: s2.ioWrite,
})
}
byCPU := topN(merged, limit, func(a, b ProcessStat) bool { return a.CPUPercent > b.CPUPercent })
byMem := topN(merged, limit, func(a, b ProcessStat) bool { return a.RSSBytes > b.RSSBytes })
byIO := topN(merged, limit, func(a, b ProcessStat) bool {
return (a.IOReadTot + a.IOWriteTot) > (b.IOReadTot + b.IOWriteTot)
})
return TopResponse{
Timestamp: time.Now().UTC(),
SampleMs: sampleWindow.Milliseconds(),
TotalProcs: len(merged),
TopByCPU: byCPU,
TopByMemory: byMem,
TopByIO: byIO,
}, nil
}
type procSample struct {
pid int
command string
utime uint64
stime uint64
vsize uint64
rssPages uint64
uid int
ioRead uint64
ioWrite uint64
}
func snapshotProcesses() (map[int]procSample, error) {
entries, err := os.ReadDir("/proc")
if err != nil {
return nil, err
}
out := make(map[int]procSample, 256)
for _, e := range entries {
if !e.IsDir() {
continue
}
pid, err := strconv.Atoi(e.Name())
if err != nil || pid <= 0 {
continue
}
s, ok := readProcSample(pid)
if !ok {
continue
}
out[pid] = s
}
return out, nil
}
func readProcSample(pid int) (procSample, bool) {
pidStr := strconv.Itoa(pid)
statData, err := os.ReadFile("/proc/" + pidStr + "/stat")
if err != nil {
return procSample{}, false
}
// comm is in parens and may contain spaces; parse after the last ')'.
line := string(statData)
rp := strings.LastIndexByte(line, ')')
if rp <= 0 {
return procSample{}, false
}
lp := strings.IndexByte(line, '(')
if lp < 0 || lp >= rp {
return procSample{}, false
}
comm := line[lp+1 : rp]
rest := strings.Fields(line[rp+2:])
// After comm and the state char, indices inside `rest`:
// 0: state ... but we split from after ') '. fields[0]=state, fields[1]=ppid,
// fields[2]=pgrp, ..., fields[11]=utime, fields[12]=stime, fields[19]=vsize, fields[20]=rss.
if len(rest) < 22 {
return procSample{}, false
}
utime, _ := strconv.ParseUint(rest[11], 10, 64)
stime, _ := strconv.ParseUint(rest[12], 10, 64)
vsize, _ := strconv.ParseUint(rest[20], 10, 64)
rssPages, _ := strconv.ParseUint(rest[21], 10, 64)
uid := readProcUID(pidStr)
ioRead, ioWrite := readProcIO(pidStr)
return procSample{
pid: pid,
command: comm,
utime: utime,
stime: stime,
vsize: vsize,
rssPages: rssPages,
uid: uid,
ioRead: ioRead,
ioWrite: ioWrite,
}, true
}
func readProcUID(pidStr string) int {
data, err := os.ReadFile("/proc/" + pidStr + "/status")
if err != nil {
return -1
}
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "Uid:") {
fields := strings.Fields(line)
if len(fields) >= 2 {
if v, err := strconv.Atoi(fields[1]); err == nil {
return v
}
}
return -1
}
}
return -1
}
func readProcIO(pidStr string) (uint64, uint64) {
data, err := os.ReadFile("/proc/" + pidStr + "/io")
if err != nil {
return 0, 0
}
var rb, wb uint64
for _, line := range strings.Split(string(data), "\n") {
parts := strings.SplitN(line, ":", 2)
if len(parts) != 2 {
continue
}
key := strings.TrimSpace(parts[0])
val := strings.TrimSpace(parts[1])
v, _ := strconv.ParseUint(val, 10, 64)
switch key {
case "read_bytes":
rb = v
case "write_bytes":
wb = v
}
}
return rb, wb
}
var cachedClockTicks float64
var cachedClockOnce sync.Once
func clockTicksPerSecond() float64 {
cachedClockOnce.Do(func() {
// SC_CLK_TCK is almost always 100 on Linux; read from /proc/self/stat
// + uptime as a sanity check would be nicer but we accept the default.
cachedClockTicks = 100.0
})
return cachedClockTicks
}
type usernameCache struct {
cache map[int]string
}
func newUsernameCache() *usernameCache {
return &usernameCache{cache: make(map[int]string, 16)}
}
func (c *usernameCache) lookup(uid int) string {
if uid < 0 {
return "-"
}
if v, ok := c.cache[uid]; ok {
return v
}
u, err := user.LookupId(strconv.Itoa(uid))
if err != nil || u == nil {
name := strconv.Itoa(uid)
c.cache[uid] = name
return name
}
c.cache[uid] = u.Username
return u.Username
}
func topN(in []ProcessStat, n int, less func(a, b ProcessStat) bool) []ProcessStat {
cp := make([]ProcessStat, len(in))
copy(cp, in)
sort.Slice(cp, func(i, j int) bool { return less(cp[i], cp[j]) })
if len(cp) > n {
cp = cp[:n]
}
return cp
}
// DirStat is one directory entry in a du-style listing.
type DirStat struct {
Path string `json:"path"`
Name string `json:"name"`
Bytes uint64 `json:"bytes"`
Files uint64 `json:"files"`
}
// DUResponse is returned by /api/v1/du.
type DUResponse struct {
Timestamp time.Time `json:"timestamp"`
Root string `json:"root"`
Entries []DirStat `json:"entries"`
Truncated bool `json:"truncated"`
}
// CollectDirSizes lists immediate children of root (only directories) and
// sums file sizes under each child recursively, respecting ctx deadline.
// Returned entries are sorted by Bytes descending. Truncated is true if the
// walker stopped early due to timeout.
func CollectDirSizes(ctx context.Context, root string, limit int) (DUResponse, error) {
root = strings.TrimSpace(root)
if root == "" {
root = "/"
}
absRoot, err := filepath.Abs(root)
if err != nil {
return DUResponse{}, err
}
info, err := os.Stat(absRoot)
if err != nil {
return DUResponse{}, err
}
if !info.IsDir() {
return DUResponse{}, errors.New("root is not a directory")
}
if limit <= 0 || limit > 100 {
limit = 15
}
entries, err := os.ReadDir(absRoot)
if err != nil {
return DUResponse{}, err
}
truncated := false
out := make([]DirStat, 0, len(entries))
for _, e := range entries {
if ctx.Err() != nil {
truncated = true
break
}
if !e.IsDir() {
continue
}
name := e.Name()
// Skip virtual filesystems when rooted at /.
if absRoot == "/" && isSkippedSystemDir(name) {
continue
}
full := filepath.Join(absRoot, name)
size, files, stopped := sumDirectory(ctx, full)
if stopped {
truncated = true
}
out = append(out, DirStat{
Path: full,
Name: name,
Bytes: size,
Files: files,
})
if stopped {
break
}
}
sort.Slice(out, func(i, j int) bool { return out[i].Bytes > out[j].Bytes })
if len(out) > limit {
out = out[:limit]
}
return DUResponse{
Timestamp: time.Now().UTC(),
Root: absRoot,
Entries: out,
Truncated: truncated,
}, nil
}
func isSkippedSystemDir(name string) bool {
switch name {
case "proc", "sys", "dev", "run", "tmp":
return true
}
return false
}
func sumDirectory(ctx context.Context, path string) (uint64, uint64, bool) {
var total uint64
var files uint64
stopped := false
walkFn := func(p string, d fs.DirEntry, err error) error {
if err != nil {
if d != nil && d.IsDir() {
return filepath.SkipDir
}
return nil
}
if ctx.Err() != nil {
stopped = true
return filepath.SkipAll
}
if d.IsDir() {
// Skip known pseudo filesystems we may cross into.
name := d.Name()
if p != path && (name == "proc" || name == "sys" || name == "dev") {
return filepath.SkipDir
}
return nil
}
info, err := d.Info()
if err != nil {
return nil
}
total += uint64(info.Size())
files++
return nil
}
_ = filepath.WalkDir(path, walkFn)
return total, files, stopped
}
// scanLinesToFields is a tiny helper for tests and future use.
func scanLinesToFields(data []byte) [][]string {
var out [][]string
sc := bufio.NewScanner(strings.NewReader(string(data)))
for sc.Scan() {
out = append(out, strings.Fields(sc.Text()))
}
return out
}
internal/cli/app.go
+2969
View File
File diff suppressed because it is too large Load Diff
internal/cli/bot.go
+1523
View File
File diff suppressed because it is too large Load Diff
internal/cli/bot_usage.go
+440
View File
@@ -0,0 +1,440 @@
package cli
import (
"bytes"
"context"
"fmt"
"io"
"mime/multipart"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"pxmon/internal/cluster"
"pxmon/internal/history"
)
type samplerIfaceCounter struct {
RxBytes uint64
TxBytes uint64
}
type samplerState struct {
Timestamp time.Time
Counters map[string]samplerIfaceCounter
}
var (
samplerMu sync.Mutex
samplerPrev = map[string]*samplerState{}
)
// handleBotUsageCommand intercepts usage/traffic/graph commands before they
// reach the generic runObserverScopedCommand bridge so it can send richer
// payloads (including PNG attachments) back to Telegram. Returns true when
// the command was consumed.
func handleBotUsageCommand(ctx context.Context, client *http.Client, svc *cluster.Service, token string, chatID int64, cmd string) bool {
fields := strings.Fields(cmd)
if len(fields) == 0 {
return false
}
rest := fields
if strings.EqualFold(fields[0], "cluster") && len(fields) > 1 {
rest = fields[1:]
}
head := strings.ToLower(rest[0])
if head != "usage" && head != "traffic" && head != "graph" && head != "p95" {
return false
}
selector := ""
rangeStr := defaultBotRange(head)
iface := ""
for _, f := range rest[1:] {
if strings.HasPrefix(f, "--iface=") {
iface = strings.TrimSpace(strings.TrimPrefix(f, "--iface="))
continue
}
if strings.HasPrefix(f, "--range=") {
rangeStr = strings.TrimPrefix(f, "--range=")
continue
}
if rng, ok := history.ParseRangeShortcut(f); ok {
rangeStr = string(rng)
continue
}
if selector == "" {
if head == "p95" && iface == "" && !strings.HasPrefix(f, "-") {
iface = f
} else {
selector = f
}
}
}
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(rangeStr))
if !ok {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>%s</b>\n<pre>invalid range %q</pre>", head, rangeStr))
return true
}
gather, cancel := context.WithTimeout(ctx, 50*time.Second)
defer cancel()
snap, err := svc.CollectUsageSnapshot(gather, selector, rng, "/")
if err != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>%s</b>\n<pre>%s</pre>", head, htmlEscapeTelegram(err.Error())))
return true
}
switch head {
case "usage":
body := formatUsageForBot(snap, rng)
header := fmt.Sprintf("🟢 <b>usage %s</b> <i>%s</i>", htmlEscapeTelegram(snap.ClusterName), rng.Label())
for _, chunk := range splitTelegramText(htmlEscapeTelegram(body), 3400) {
_ = telegramSendHTML(ctx, client, token, chatID, header+"\n<pre>"+chunk+"</pre>")
header = ""
}
case "traffic":
body := formatTrafficForBot(snap, rng)
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🟢 <b>traffic %s</b> <i>%s</i>\n<pre>%s</pre>",
htmlEscapeTelegram(snap.ClusterName), rng.Label(), htmlEscapeTelegram(body)))
case "graph":
png, err := cluster.RenderUsageChartPNG(snap, "")
if err != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>graph</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
return true
}
caption := fmt.Sprintf("📈 <b>%s</b> <i>%s</i>\nP95: %s · max: %s · avg: %s · %d samples",
htmlEscapeTelegram(snap.ClusterName),
rng.Label(),
formatMbpsHuman(snap.P95TotalMbps),
formatMbpsHuman(snap.MaxTotalMbps),
formatMbpsHuman(snap.AvgTotalMbps),
len(snap.NodeSeries),
)
filename := sanitizeFilename(snap.ClusterName) + "-" + string(rng) + ".png"
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>graph</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
}
case "p95":
if strings.TrimSpace(iface) == "" {
_ = telegramSendHTML(ctx, client, token, chatID, "🔴 <b>p95</b>\n<pre>iface is required (example: p95 eth0 30d)</pre>")
return true
}
p95Snap, err := svc.CollectInterfaceP95(selector, iface, rng)
if err != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>p95</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
return true
}
png, err := svc.RenderInterfaceP95GraphPNG(p95Snap)
if err != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>p95</b>\n<pre>render failed: %s</pre>", htmlEscapeTelegram(err.Error())))
return true
}
filename := sanitizeFilename(p95Snap.ClusterName) + "-" + sanitizeFilename(p95Snap.Interface) + "-" + string(rng) + ".png"
caption := fmt.Sprintf("📈 <b>P95 %s</b> <i>%s</i>\niface: %s\nP95: %s · max: %s · avg: %s · %d samples",
htmlEscapeTelegram(p95Snap.ClusterName),
rng.Label(),
htmlEscapeTelegram(p95Snap.Interface),
formatMbpsHuman(p95Snap.P95Mbps),
formatMbpsHuman(p95Snap.MaxMbps),
formatMbpsHuman(p95Snap.AvgMbps),
p95Snap.Samples,
)
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
_ = telegramSendHTML(ctx, client, token, chatID,
fmt.Sprintf("🔴 <b>p95</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
}
}
return true
}
func defaultBotRange(cmd string) string {
switch cmd {
case "usage":
return "live"
case "traffic":
return "1h"
case "graph":
return "1d"
case "p95":
return "30d"
}
return "live"
}
// trySendGraphAttachmentFromCLIOutput is a safety net for graph commands that
// were executed through the generic CLI bridge and returned text like:
//
// Saved: <file.png>
//
// It uploads the generated PNG so Telegram users always receive the image.
func trySendGraphAttachmentFromCLIOutput(ctx context.Context, client *http.Client, token string, chatID int64, cmd, out string) (bool, error) {
fields := strings.Fields(strings.TrimSpace(cmd))
if len(fields) == 0 {
return false, nil
}
isGraph := false
if strings.EqualFold(fields[0], "graph") {
isGraph = true
}
if len(fields) >= 2 && strings.EqualFold(fields[0], "cluster") && strings.EqualFold(fields[1], "graph") {
isGraph = true
}
if !isGraph {
return false, nil
}
lines := strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n")
saved := ""
p95 := ""
for _, line := range lines {
t := strings.TrimSpace(stripANSI(line))
if strings.HasPrefix(strings.ToLower(t), "saved:") {
saved = strings.TrimSpace(t[len("saved:"):])
continue
}
if strings.HasPrefix(strings.ToLower(t), "p95:") {
p95 = t
}
}
if saved == "" {
return false, nil
}
data, err := os.ReadFile(saved)
if err != nil {
return false, fmt.Errorf("read graph png %q: %w", saved, err)
}
filename := filepath.Base(saved)
caption := "📈 <b>cluster graph</b>"
if p95 != "" {
caption += "\n" + htmlEscapeTelegram(p95)
}
if err := telegramSendPhoto(ctx, client, token, chatID, filename, data, caption); err != nil {
return false, err
}
return true, nil
}
func formatTrafficForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string {
var b strings.Builder
fmt.Fprintf(&b, "range: %s\n", rng.Label())
fmt.Fprintf(&b, "samples: %d\n", len(snap.NodeSeries))
fmt.Fprintf(&b, "P95: %s\n", formatMbpsHuman(snap.P95TotalMbps))
fmt.Fprintf(&b, "max: %s\n", formatMbpsHuman(snap.MaxTotalMbps))
fmt.Fprintf(&b, "avg: %s\n", formatMbpsHuman(snap.AvgTotalMbps))
if snap.TopIfaceName != "" {
fmt.Fprintf(&b, "uplink: %s (avg %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps))
}
if snap.HistoryError != "" {
fmt.Fprintf(&b, "note: %s\n", snap.HistoryError)
}
return b.String()
}
// telegramSendPhoto uploads a PNG to Telegram via multipart sendPhoto.
func telegramSendPhoto(ctx context.Context, client *http.Client, token string, chatID int64, filename string, png []byte, captionHTML string) error {
var buf bytes.Buffer
w := multipart.NewWriter(&buf)
if err := w.WriteField("chat_id", strconv.FormatInt(chatID, 10)); err != nil {
return err
}
if captionHTML != "" {
if err := w.WriteField("caption", captionHTML); err != nil {
return err
}
if err := w.WriteField("parse_mode", "HTML"); err != nil {
return err
}
}
part, err := w.CreateFormFile("photo", filename)
if err != nil {
return err
}
if _, err := part.Write(png); err != nil {
return err
}
if err := w.Close(); err != nil {
return err
}
endpoint := "https://api.telegram.org/bot" + token + "/sendPhoto"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, &buf)
if err != nil {
return err
}
req.Header.Set("Content-Type", w.FormDataContentType())
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 300 {
body, _ := io.ReadAll(resp.Body)
msg := strings.TrimSpace(string(body))
if len(msg) > 200 {
msg = msg[:200]
}
return fmt.Errorf("sendPhoto HTTP %d: %s", resp.StatusCode, msg)
}
return nil
}
// runHistorySampler periodically appends a network snapshot for every
// cluster that has an agent installed so long-running non-TUI processes
// (like the bot daemon) can build up data for P95/graph queries.
func runHistorySampler(ctx context.Context, svc *cluster.Service, interval time.Duration, logErr io.Writer) {
if svc == nil || svc.NetworkStore() == nil {
return
}
if interval <= 0 {
interval = 30 * time.Second
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
sampleOnce(ctx, svc, logErr)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
sampleOnce(ctx, svc, logErr)
}
}
}
func sampleOnce(ctx context.Context, svc *cluster.Service, logErr io.Writer) {
clusters, _, err := svc.List()
if err != nil {
return
}
store := svc.NetworkStore()
if store == nil {
return
}
availStore := history.NewAvailabilityStore(svc.DataDir())
capStore := history.NewCapacityStore(svc.DataDir())
for _, c := range clusters {
if !c.Agent.Installed {
continue
}
select {
case <-ctx.Done():
return
default:
}
callCtx, cancel := context.WithTimeout(ctx, 6*time.Second)
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
cancel()
if err != nil {
fmt.Fprintf(logErr, "history sampler: %s: %v\n", c.Name, err)
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
Timestamp: time.Now().UTC(),
ClusterUp: false,
Error: err.Error(),
})
continue
}
items := make([]history.InterfaceSample, 0, len(stats.Network))
// With a single stats point we don't have rate info; estimate Mbps
// per-interface as a simple counter delta versus the sampler's prior
// snapshot if available.
samplerMu.Lock()
prev := samplerPrev[c.ID]
samplerMu.Unlock()
now := time.Now().UTC()
for _, n := range stats.Network {
rxMbps := 0.0
txMbps := 0.0
if prev != nil {
if pn, ok := prev.Counters[n.Interface]; ok {
elapsed := now.Sub(prev.Timestamp).Seconds()
if elapsed > 0.1 {
if n.RxBytes >= pn.RxBytes {
rxMbps = float64(n.RxBytes-pn.RxBytes) * 8 / elapsed / 1_000_000
}
if n.TxBytes >= pn.TxBytes {
txMbps = float64(n.TxBytes-pn.TxBytes) * 8 / elapsed / 1_000_000
}
}
}
}
items = append(items, history.InterfaceSample{
Interface: n.Interface,
RxMbps: rxMbps,
TxMbps: txMbps,
RxDrops: n.RxDrops,
TxDrops: n.TxDrops,
})
}
counters := make(map[string]samplerIfaceCounter, len(stats.Network))
for _, n := range stats.Network {
counters[n.Interface] = samplerIfaceCounter{RxBytes: n.RxBytes, TxBytes: n.TxBytes}
}
samplerMu.Lock()
samplerPrev[c.ID] = &samplerState{Timestamp: now, Counters: counters}
samplerMu.Unlock()
// Only persist when we have a real delta — skip the first bootstrap
// sample per cluster to avoid a meaningless zero row.
if prev != nil {
snap := history.NetworkSnapshot{
Timestamp: now,
Interfaces: items,
}
if err := store.Append(c.ID, snap); err != nil {
fmt.Fprintf(logErr, "history sampler: append %s: %v\n", c.Name, err)
}
}
// Availability and VM state snapshot.
vmStates := map[string]string{}
vmCtx, vmCancel := context.WithTimeout(ctx, 6*time.Second)
if rawStates, vmErr := svc.ListVMStates(vmCtx, c.ID); vmErr == nil {
vmStates = rawStates
}
vmCancel()
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
Timestamp: time.Now().UTC(),
ClusterUp: true,
VMStates: vmStates,
})
// Capacity snapshot (disk usage trend source).
disks := make([]history.CapacityDiskPoint, 0, len(stats.Disk))
for _, d := range stats.Disk {
if d.TotalBytes == 0 {
continue
}
disks = append(disks, history.CapacityDiskPoint{
Mount: d.MountPoint,
UsedBytes: d.UsedBytes,
TotalBytes: d.TotalBytes,
})
}
if len(disks) > 0 {
_ = capStore.Append(c.ID, history.CapacitySnapshot{
Timestamp: time.Now().UTC(),
Disks: disks,
})
}
}
}
internal/cli/command_exec.go
+182
View File
@@ -0,0 +1,182 @@
package cli
import (
"context"
"errors"
"os/exec"
"regexp"
"strings"
"time"
"pxmon/internal/cluster"
)
type observerCommandOptions struct {
AllowShellEscape bool
StatsAutoOnce bool
BlockBotRun bool
StripANSI bool
EmbeddedConsole bool
}
var ansiEscapeRE = regexp.MustCompile(`\x1b\[[0-9;?]*[ -/]*[@-~]`)
func stripANSI(s string) string {
if s == "" {
return s
}
return ansiEscapeRE.ReplaceAllString(s, "")
}
func runObserverScopedCommand(svc *cluster.Service, configPath, line string, opts observerCommandOptions) (string, int) {
out, code := runObserverScopedCommandInner(svc, configPath, line, opts)
if opts.StripANSI {
out = stripANSI(out)
}
return out, code
}
func runObserverScopedCommandInner(svc *cluster.Service, configPath, line string, opts observerCommandOptions) (string, int) {
line = strings.TrimSpace(line)
if line == "" {
return "empty command", 2
}
if strings.HasPrefix(line, "!") {
if !opts.AllowShellEscape {
return "shell escape is disabled for this channel", 2
}
cmdline := strings.TrimSpace(strings.TrimPrefix(line, "!"))
if cmdline == "" {
return "usage: !<shell command>", 2
}
cmd := exec.Command("/bin/sh", "-lc", cmdline)
out, err := cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) {
return string(out), exitErr.ExitCode()
}
if len(out) == 0 {
return err.Error(), 1
}
return string(out), 1
}
return string(out), 0
}
args, err := parseShellArgs(line)
if err != nil || len(args) == 0 {
if err != nil {
return err.Error(), 2
}
return "empty command", 2
}
norm := normalizeObserverConsoleArgs(args)
if len(norm) == 0 {
return "empty command", 2
}
if opts.BlockBotRun && len(norm) >= 3 &&
strings.EqualFold(norm[0], "bot") &&
strings.EqualFold(norm[1], "telegram") &&
strings.EqualFold(norm[2], "run") {
return "running `bot telegram run` from bot channel is blocked", 2
}
if opts.StatsAutoOnce && len(norm) >= 2 &&
strings.EqualFold(norm[0], "cluster") &&
strings.EqualFold(norm[1], "stats") &&
!hasArg(norm[2:], "--once") {
norm = append(norm, "--once")
}
if handled, out, code := runPluginArgs(svc, norm); handled {
return out, code
}
return runObserverCommand(configPath, norm, opts.EmbeddedConsole)
}
func runPluginArgs(svc *cluster.Service, args []string) (bool, string, int) {
if len(args) == 0 || svc == nil {
return false, "", 0
}
tool := strings.ToLower(strings.TrimSpace(args[0]))
switch tool {
case "kvm", "lxc", "lxd", "bird", "frr":
default:
return false, "", 0
}
selector, rest, err := parseClusterSelectorArg(args[1:])
if err != nil {
return true, err.Error(), 2
}
action := ""
params := []string{}
if len(rest) > 0 {
action = strings.ToLower(strings.TrimSpace(rest[0]))
params = rest[1:]
}
if action == "" {
switch tool {
case "kvm", "lxc", "lxd":
action = "list"
default:
action = "status"
}
}
if tool == "kvm" && action == "top" {
for _, p := range params {
if strings.EqualFold(strings.TrimSpace(p), "--live") || strings.EqualFold(strings.TrimSpace(p), "-L") {
return true, "kvm top --live is removed; use `kvm top` for allocated VM specs", 2
}
}
}
ctx, cancel := context.WithTimeout(context.Background(), pluginActionTimeout(tool, action, false))
defer cancel()
out, execErr := svc.RunPluginAction(ctx, selector, tool, action, params)
if execErr != nil {
return true, execErr.Error(), 1
}
if strings.TrimSpace(out) == "" {
out = "ok"
}
return true, out, 0
}
func pluginActionTimeout(tool, action string, live bool) time.Duration {
tool = strings.ToLower(strings.TrimSpace(tool))
action = strings.ToLower(strings.TrimSpace(action))
switch tool {
case "kvm":
switch action {
case "top":
if live {
return 2 * time.Minute
}
return 90 * time.Second
case "net-top", "net":
if live {
return 90 * time.Second
}
return 60 * time.Second
default:
return 45 * time.Second
}
case "lxc", "lxd":
if action == "top" || action == "net-top" || action == "net" {
return 45 * time.Second
}
}
if live {
return 35 * time.Second
}
return 25 * time.Second
}
internal/cli/extras.go
+2068
View File
File diff suppressed because it is too large Load Diff
internal/cli/monitor.go
+5508
View File
File diff suppressed because it is too large Load Diff
internal/cli/monitor_live.go
+497
View File
@@ -0,0 +1,497 @@
package cli
import (
"context"
"fmt"
"sort"
"strings"
"sync"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"pxmon/internal/cluster"
"pxmon/internal/history"
)
// liveClusterStat is one row on the live dashboard. Fields are best-effort;
// if the agent is unreachable we still render the row with an error so the
// operator sees the node exists but is degraded.
type liveClusterStat struct {
ClusterID string
Name string
Host string
Reachable bool
Err string
CPUPercent float64
MemPercent float64
MemUsed uint64
MemTotal uint64
DiskUsePct float64
NetMbps float64 // current: last point from history
P95Mbps float64 // primary iface, 24h window
Uplink string
HostName string
}
type monitorLiveMsg struct {
entries []liveClusterStat
err error
}
type liveAutoTickMsg struct{}
func liveAutoTickCmd() tea.Cmd {
return tea.Tick(liveAutoTickInterval, func(_ time.Time) tea.Msg {
return liveAutoTickMsg{}
})
}
const (
livePageSize = 14
liveFetchWorkers = 16
liveAutoTickInterval = 5 * time.Second
liveP95HistoryWindow = 24 * time.Hour
liveFetchClusterTO = 5 * time.Second
liveFetchOverallTO = 25 * time.Second
)
// fetchLiveCmd gathers live stats for every cluster with an installed agent
// in parallel, pairs each with P95 from history, and emits a sorted slice.
func (m monitorModel) fetchLiveCmd() tea.Cmd {
svc := m.svc
return func() tea.Msg {
clusters, _, err := svc.List()
if err != nil {
return monitorLiveMsg{err: err}
}
ctx, cancel := context.WithTimeout(context.Background(), liveFetchOverallTO)
defer cancel()
type work struct {
idx int
c cluster.Cluster
}
ch := make(chan work, len(clusters))
results := make([]liveClusterStat, len(clusters))
var wg sync.WaitGroup
workers := liveFetchWorkers
if workers > len(clusters) {
workers = len(clusters)
}
if workers < 1 {
workers = 1
}
for w := 0; w < workers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for item := range ch {
results[item.idx] = collectLiveRow(ctx, svc, item.c)
}
}()
}
for i, c := range clusters {
ch <- work{idx: i, c: c}
}
close(ch)
wg.Wait()
out := make([]liveClusterStat, 0, len(results))
for _, r := range results {
if r.ClusterID == "" {
continue
}
out = append(out, r)
}
return monitorLiveMsg{entries: out}
}
}
func collectLiveRow(ctx context.Context, svc *cluster.Service, c cluster.Cluster) liveClusterStat {
row := liveClusterStat{
ClusterID: c.ID,
Name: c.Name,
Host: c.Host,
}
if !c.Agent.Installed {
row.Err = "agent not installed"
return row
}
callCtx, cancel := context.WithTimeout(ctx, liveFetchClusterTO)
defer cancel()
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
if err != nil {
row.Err = err.Error()
return row
}
row.Reachable = true
row.CPUPercent = stats.CPU.UsagePercent
row.MemPercent = stats.Memory.UsedPercent
row.MemUsed = stats.Memory.UsedBytes
row.MemTotal = stats.Memory.TotalBytes
row.HostName = stats.Host.Hostname
var maxDisk float64
for _, d := range stats.Disk {
if d.UsedPercent > maxDisk {
maxDisk = d.UsedPercent
}
}
row.DiskUsePct = maxDisk
// Historical P95 from the persisted store (populated by the sampler
// running inside TUI or bot daemon).
if store := svc.NetworkStore(); store != nil {
since := time.Now().Add(-liveP95HistoryWindow)
if snaps, err := store.Load(c.ID, since); err == nil && len(snaps) > 0 {
primary := history.PrimaryInterface(snaps)
row.Uplink = primary
series := history.AggregateNodeSeries(snaps, primary)
if len(series) > 0 {
row.P95Mbps = history.PercentileMbps(series, 95)
row.NetMbps = series[len(series)-1].TotalMbps
}
}
}
// Fallback / live reading: take a second stats snapshot after a short
// pause and derive the instantaneous per-interface Mbps ourselves.
// This makes NET Mbps meaningful even when the history store is empty
// (e.g. the sampler hasn't been running long).
time.Sleep(700 * time.Millisecond)
callCtx2, cancel2 := context.WithTimeout(ctx, liveFetchClusterTO)
defer cancel2()
stats2, err := svc.AgentStatsTyped(callCtx2, c.ID)
if err != nil {
return row
}
elapsed := stats2.Timestamp.Sub(stats.Timestamp).Seconds()
if elapsed <= 0 {
elapsed = 0.7
}
prev := make(map[string]struct {
rx, tx uint64
}, len(stats.Network))
for _, n := range stats.Network {
prev[n.Interface] = struct{ rx, tx uint64 }{n.RxBytes, n.TxBytes}
}
bestName := ""
bestMbps := -1.0
for _, n := range stats2.Network {
if isLiveVirtual(n.Interface) {
continue
}
p, ok := prev[n.Interface]
if !ok {
continue
}
var rxB, txB uint64
if n.RxBytes >= p.rx {
rxB = n.RxBytes - p.rx
}
if n.TxBytes >= p.tx {
txB = n.TxBytes - p.tx
}
rxMbps := float64(rxB) * 8 / elapsed / 1_000_000
txMbps := float64(txB) * 8 / elapsed / 1_000_000
m := rxMbps
if txMbps > m {
m = txMbps
}
if m > bestMbps {
bestMbps = m
bestName = n.Interface
}
}
if bestMbps >= 0 {
row.NetMbps = bestMbps
if row.Uplink == "" {
row.Uplink = bestName
}
}
return row
}
// isLiveVirtual mirrors history.isVirtualIface but is local so we don't
// export the original.
func isLiveVirtual(name string) bool {
n := strings.ToLower(name)
if n == "" || n == "lo" {
return true
}
prefixes := []string{
"lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr",
"cni", "flannel", "wg", "tun", "tailscale", "zt", "kube",
"cilium", "ovs", "podman", "dummy",
}
for _, p := range prefixes {
if strings.HasPrefix(n, p) {
return true
}
}
return false
}
// handleLiveKey routes keys while the live dashboard is visible.
func (m monitorModel) handleLiveKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
switch v.String() {
case "o", "esc":
m.view = viewOverview
return m, nil
case "c":
m.view = viewClusters
return m, nil
case "r", "R":
m.liveLoading = true
m.setStatus("live: refreshing")
return m, m.fetchLiveCmd()
case "up", "k":
if m.liveCursor > 0 {
m.liveCursor--
}
return m, nil
case "down", "j":
if m.liveCursor < len(m.liveEntries)-1 {
m.liveCursor++
}
return m, nil
case "pgup":
if m.livePage > 0 {
m.livePage--
}
return m, nil
case "pgdown":
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
if m.livePage < pages-1 {
m.livePage++
}
return m, nil
case "home":
m.liveCursor = 0
m.livePage = 0
return m, nil
case "end":
m.liveCursor = len(m.liveEntries) - 1
if m.liveCursor < 0 {
m.liveCursor = 0
}
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
if pages > 0 {
m.livePage = pages - 1
}
return m, nil
case "enter":
if m.livePinned == nil {
m.livePinned = map[string]bool{}
}
rows := m.sortedLiveEntries()
if m.liveCursor >= 0 && m.liveCursor < len(rows) {
id := rows[m.liveCursor].ClusterID
if m.livePinned[id] {
delete(m.livePinned, id)
m.setStatus("live: unpinned " + rows[m.liveCursor].Name)
} else {
m.livePinned[id] = true
m.setStatus("live: pinned " + rows[m.liveCursor].Name)
}
}
return m, nil
case "1":
m.liveSort = 0
return m, nil
case "2":
m.liveSort = 1
return m, nil
case "3":
m.liveSort = 2
return m, nil
case "4":
m.liveSort = 3
return m, nil
case "5":
m.liveSort = 4
return m, nil
}
return m, nil
}
// sortedLiveEntries returns entries sorted by the active metric. Pinned
// clusters always float to the top so the operator keeps eyes on them even
// when their usage changes.
func (m monitorModel) sortedLiveEntries() []liveClusterStat {
entries := make([]liveClusterStat, len(m.liveEntries))
copy(entries, m.liveEntries)
key := func(r liveClusterStat) float64 {
switch m.liveSort {
case 1:
return r.MemPercent
case 2:
return r.NetMbps
case 3:
return r.P95Mbps
case 4:
return r.DiskUsePct
default:
return r.CPUPercent
}
}
sort.SliceStable(entries, func(i, j int) bool {
pi := m.livePinned[entries[i].ClusterID]
pj := m.livePinned[entries[j].ClusterID]
if pi != pj {
return pi
}
return key(entries[i]) > key(entries[j])
})
return entries
}
func liveSortLabel(n int) string {
switch n {
case 1:
return "ram"
case 2:
return "net"
case 3:
return "p95"
case 4:
return "disk"
default:
return "cpu"
}
}
// renderLiveDashboard paints the paginated live view.
func (m monitorModel) renderLiveDashboard(width int) string {
title := titleStyle.Render("━━ Live cluster fleet ━━")
help := dimStyle.Render("sort: 1=cpu 2=ram 3=net 4=p95 5=disk enter pin r refresh ↑/↓ select pgup/pgdn page o back")
if m.liveLoading && len(m.liveEntries) == 0 {
body := dimStyle.Render("collecting stats from every cluster…")
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
}
if m.liveError != "" && len(m.liveEntries) == 0 {
body := critStyle.Render("error: " + m.liveError)
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
}
if len(m.liveEntries) == 0 {
return lipgloss.JoinVertical(lipgloss.Left, title, help, "",
dimStyle.Render("no clusters with installed agents"))
}
entries := m.sortedLiveEntries()
total := len(entries)
pages := (total + livePageSize - 1) / livePageSize
if pages == 0 {
pages = 1
}
if m.livePage >= pages {
m.livePage = pages - 1
}
start := m.livePage * livePageSize
end := start + livePageSize
if end > total {
end = total
}
page := entries[start:end]
headerLine := fmt.Sprintf(" %-3s %-16s %-3s %6s %6s %14s %12s %12s %6s %-10s",
"#", "NAME", "PIN", "CPU%", "RAM%", "RAM", "NET", "P95", "DISK%", "UPLINK")
lines := []string{title, help, "",
dimStyle.Render(fmt.Sprintf("sorted by %s · page %d/%d · %d clusters · refresh every %s · enter=pin",
liveSortLabel(m.liveSort), m.livePage+1, pages, total, liveAutoTickInterval)),
"",
accentStyle.Bold(true).Render(headerLine),
}
for i, row := range page {
absoluteIdx := start + i
selected := absoluteIdx == m.liveCursor
pin := " "
if m.livePinned[row.ClusterID] {
pin = "⚑"
}
name := row.Name
if len(name) > 16 {
name = name[:13] + "..."
}
if !row.Reachable {
line := fmt.Sprintf(" %-3d %-16s %-3s %s",
absoluteIdx+1, name, pin, critStyle.Render("unreachable: "+clipRight(row.Err, 60)))
if selected {
line = selectedRowStyle.Render(line)
}
lines = append(lines, line)
continue
}
cpuStr := colorByThreshold(fmt.Sprintf("%5.1f", row.CPUPercent), row.CPUPercent, 50, 80)
ramStr := colorByThreshold(fmt.Sprintf("%5.1f", row.MemPercent), row.MemPercent, 50, 80)
ramAbs := fmt.Sprintf("%7s/%-6s", humanBytesUint(row.MemUsed), humanBytesUint(row.MemTotal))
netStr := fmt.Sprintf("%12s", formatMbpsHuman(row.NetMbps))
var p95Str string
if row.P95Mbps > 0 {
p95Str = fmt.Sprintf("%12s", formatMbpsHuman(row.P95Mbps))
} else {
p95Str = dimStyle.Render(fmt.Sprintf("%12s", "— (no hist)"))
}
diskStr := colorByThreshold(fmt.Sprintf("%5.1f", row.DiskUsePct), row.DiskUsePct, 70, 90)
uplink := row.Uplink
if uplink == "" {
uplink = "-"
}
if len(uplink) > 10 {
uplink = uplink[:10]
}
body := fmt.Sprintf(" %-3d %-16s %-3s %s %s %14s %s %s %s %-10s",
absoluteIdx+1, name, pin,
cpuStr, ramStr, ramAbs, netStr, p95Str, diskStr, uplink)
if selected {
body = selectedRowStyle.Render(body)
}
lines = append(lines, body)
}
// Stats footer.
var totalCPU, totalMem, totalNet, totalP95 float64
reachable := 0
for _, r := range entries {
if !r.Reachable {
continue
}
reachable++
totalCPU += r.CPUPercent
totalMem += r.MemPercent
totalNet += r.NetMbps
totalP95 += r.P95Mbps
}
if reachable > 0 {
lines = append(lines, "",
dimStyle.Render(fmt.Sprintf("fleet avg: cpu %.1f%% · ram %.1f%% · net %s · p95 %s · reachable %d/%d",
totalCPU/float64(reachable),
totalMem/float64(reachable),
formatMbpsHuman(totalNet),
formatMbpsHuman(totalP95),
reachable, total)))
}
return strings.Join(lines, "\n")
}
var selectedRowStyle = lipgloss.NewStyle().Background(lipgloss.Color("#3A3A3A")).Bold(true)
func colorByThreshold(text string, value, warn, crit float64) string {
switch {
case value >= crit:
return critStyle.Render(text)
case value >= warn:
return warnStyle.Render(text)
default:
return okStyle.Render(text)
}
}
internal/cli/monitor_livecmd.go
+306
View File
@@ -0,0 +1,306 @@
package cli
import (
"context"
"fmt"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
)
// livePluginSpec describes an invocation to rerun on a ticker.
type livePluginSpec struct {
Tool string
Action string
Selector string
Params []string
Interval time.Duration
}
// Display returns a short human label for the header.
func (s livePluginSpec) Display() string {
parts := []string{s.Tool}
if s.Action != "" {
parts = append(parts, s.Action)
}
parts = append(parts, s.Params...)
out := strings.Join(parts, " ")
if s.Selector != "" {
out += " (@" + s.Selector + ")"
}
return out
}
// detectLivePluginInvocation returns a filled livePluginSpec if the parsed
// argv is a plugin command (`kvm`, `lxc`, `lxd`, `bird`, `frr`) with a
// `--live` flag. The flag is consumed so downstream execution sees a clean
// argv without it. Optional `--interval=Ns` adjusts the refresh cadence.
func detectLivePluginInvocation(args []string) (livePluginSpec, bool) {
if len(args) == 0 {
return livePluginSpec{}, false
}
tool := strings.ToLower(strings.TrimSpace(args[0]))
switch tool {
case "kvm", "lxc", "lxd", "bird", "frr":
default:
return livePluginSpec{}, false
}
hasLive := false
interval := 3 * time.Second
rest := args[1:]
clean := make([]string, 0, len(rest))
for i := 0; i < len(rest); i++ {
a := strings.TrimSpace(rest[i])
switch {
case a == "--live", a == "-L":
hasLive = true
case a == "--interval":
if i+1 < len(rest) {
if d, err := time.ParseDuration(rest[i+1]); err == nil {
interval = d
}
i++
}
case strings.HasPrefix(a, "--interval="):
if d, err := time.ParseDuration(strings.TrimPrefix(a, "--interval=")); err == nil {
interval = d
}
default:
clean = append(clean, rest[i])
}
}
if !hasLive {
return livePluginSpec{}, false
}
// Resolve the cluster selector (--cluster/-c NAME) and pull it out of
// the remaining args so action+params stay clean.
selector, cleaned2, err := parseClusterSelectorArg(clean)
if err != nil {
return livePluginSpec{}, false
}
action := ""
params := []string{}
if len(cleaned2) > 0 {
action = strings.ToLower(strings.TrimSpace(cleaned2[0]))
params = cleaned2[1:]
}
if action == "" {
switch tool {
case "kvm", "lxc", "lxd":
action = "list"
default:
action = "status"
}
}
if tool == "kvm" && action == "top" {
// kvm top is now an allocation/specs view, not a live telemetry stream.
return livePluginSpec{}, false
}
if interval < 500*time.Millisecond {
interval = 500 * time.Millisecond
}
return livePluginSpec{
Tool: tool,
Action: action,
Selector: selector,
Params: params,
Interval: interval,
}, true
}
// startLiveCmd flips the model into fullscreen live-command mode and
// schedules the first execution immediately.
func (m monitorModel) startLiveCmd(spec livePluginSpec) (tea.Model, tea.Cmd) {
m.liveCmdActive = true
m.liveCmdSpec = spec
m.liveCmdBuffer = ""
m.liveCmdErr = ""
m.liveCmdRunning = true
m.liveCmdInterval = spec.Interval
m.liveCmdScroll = 0
// Leave the pxmon console so the live view owns the screen.
m.termMode = false
m.termFull = false
m.setStatus("live: " + spec.Display())
return m, m.runLiveCmdCmd(spec)
}
// liveCmdResultMsg carries one iteration of a live-command invocation.
type liveCmdResultMsg struct {
Spec livePluginSpec
Output string
Err string
}
type liveCmdTickMsg struct{}
// runLiveCmdCmd kicks off one execution of the plugin command in a
// goroutine. The result is delivered as liveCmdResultMsg and the caller
// schedules the next tick once it lands.
func (m monitorModel) runLiveCmdCmd(spec livePluginSpec) tea.Cmd {
svc := m.svc
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), pluginActionTimeout(spec.Tool, spec.Action, true))
defer cancel()
out, err := svc.RunPluginAction(ctx, spec.Selector, spec.Tool, spec.Action, spec.Params)
res := liveCmdResultMsg{Spec: spec, Output: stripANSI(out)}
if err != nil {
res.Err = err.Error()
}
return res
}
}
func liveCmdTickCmd(d time.Duration) tea.Cmd {
if d <= 0 {
d = 3 * time.Second
}
return tea.Tick(d, func(_ time.Time) tea.Msg {
return liveCmdTickMsg{}
})
}
// handleLiveCmdKey routes keys while the live-command fullscreen is active.
func (m monitorModel) handleLiveCmdKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
switch v.String() {
case "q", "esc", "ctrl+g", "ctrl+c":
m.liveCmdActive = false
m.liveCmdBuffer = ""
m.liveCmdRunning = false
m.setStatus("live: stopped")
return m, nil
case "r", "R":
m.liveCmdRunning = true
return m, m.runLiveCmdCmd(m.liveCmdSpec)
case "+":
if m.liveCmdInterval > time.Second {
m.liveCmdInterval -= time.Second
}
m.liveCmdSpec.Interval = m.liveCmdInterval
return m, nil
case "-":
m.liveCmdInterval += time.Second
if m.liveCmdInterval > 60*time.Second {
m.liveCmdInterval = 60 * time.Second
}
m.liveCmdSpec.Interval = m.liveCmdInterval
return m, nil
case "alt+up", "up", "k":
m.liveCmdScroll += 3
return m, nil
case "alt+down", "down", "j":
m.liveCmdScroll -= 3
if m.liveCmdScroll < 0 {
m.liveCmdScroll = 0
}
return m, nil
case "alt+shift+up":
m.liveCmdScroll += 15
return m, nil
case "alt+shift+down":
m.liveCmdScroll -= 15
if m.liveCmdScroll < 0 {
m.liveCmdScroll = 0
}
return m, nil
case "pgup":
m.liveCmdScroll += 20
return m, nil
case "pgdown":
m.liveCmdScroll -= 20
if m.liveCmdScroll < 0 {
m.liveCmdScroll = 0
}
return m, nil
case "home":
return m, nil
case "end":
m.liveCmdScroll = 0
return m, nil
case "alt+p":
m.privacyMode = !m.privacyMode
return m, nil
}
return m, nil
}
// renderLiveCmdView paints the fullscreen live command view.
func (m monitorModel) renderLiveCmdView(width, height int) string {
spec := m.liveCmdSpec
title := accentStyle.Bold(true).Render(fmt.Sprintf(" live › %s ", spec.Display()))
var ageText string
if !m.liveCmdLastRun.IsZero() {
ageText = fmt.Sprintf("updated %s ago", time.Since(m.liveCmdLastRun).Round(time.Millisecond))
} else {
ageText = "pending…"
}
state := "idle"
if m.liveCmdRunning {
state = "running"
}
meta := dimStyle.Render(fmt.Sprintf(
"interval %s · %s · %s",
m.liveCmdInterval.Round(time.Second), state, ageText,
))
help := dimStyle.Render("q/esc exit · r refresh · +/- interval · alt+↑/↓ scroll · alt+shift+↑/↓ fast · alt+p privacy")
bodyLines := strings.Split(m.liveCmdBuffer, "\n")
if m.liveCmdErr != "" {
bodyLines = append([]string{critStyle.Render("error: " + m.liveCmdErr), ""}, bodyLines...)
}
if len(bodyLines) == 0 || (len(bodyLines) == 1 && bodyLines[0] == "") {
bodyLines = []string{dimStyle.Render("(no output yet)")}
}
viewportH := height - 6
if viewportH < 5 {
viewportH = 5
}
maxOffset := len(bodyLines) - viewportH
if maxOffset < 0 {
maxOffset = 0
}
if m.liveCmdScroll > maxOffset {
m.liveCmdScroll = maxOffset
}
end := len(bodyLines) - m.liveCmdScroll
start := end - viewportH
if start < 0 {
start = 0
}
if end > len(bodyLines) {
end = len(bodyLines)
}
windowed := bodyLines[start:end]
panelW := max(24, width)
bodyW := max(16, panelW-6)
for i := range windowed {
windowed[i] = truncateVisible(windowed[i], bodyW)
}
scrollBadge := ""
if m.liveCmdScroll > 0 {
scrollBadge = warnStyle.Render(fmt.Sprintf(" ↑ scrolled +%d (end to follow) ", m.liveCmdScroll))
}
panel := lipgloss.NewStyle().
BorderStyle(thinBorder).
BorderForeground(ccAccent).
Padding(0, 1).
Width(panelW).
MaxWidth(panelW).
Render(strings.Join(windowed, "\n"))
headerLine := title + " " + meta
if scrollBadge != "" {
headerLine += " " + scrollBadge
}
headerLine = truncateVisible(headerLine, panelW)
help = truncateVisible(help, panelW)
return lipgloss.JoinVertical(lipgloss.Left, headerLine, help, panel)
}
internal/cli/monitor_privacy.go
+85
View File
@@ -0,0 +1,85 @@
package cli
import (
"regexp"
"strings"
)
// Privacy mode redacts sensitive tokens (IPs, long secret-like strings,
// hostnames, MAC addresses, ssh key material) from rendered output. The
// replacement uses a shifted block pattern (▚▞) which preserves token
// length so the layout doesn't shift but makes the content obviously
// unreadable — think "frosted glass" rather than the usual `****`.
var (
privacyIPv4 = regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}\b`)
privacyIPv6 = regexp.MustCompile(`\b(?:[0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]{0,}\b`)
privacyMAC = regexp.MustCompile(`\b(?:[0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}\b`)
privacyHost = regexp.MustCompile(`\b[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}(?:\.[a-zA-Z0-9\-]{1,63}){1,}\b`)
privacyHex = regexp.MustCompile(`\b[A-Fa-f0-9]{24,}\b`)
privacyB64 = regexp.MustCompile(`\b[A-Za-z0-9+/]{28,}={0,2}\b`)
privacyKey = regexp.MustCompile(`(?i)(password|passwd|token|secret|apikey|api[_-]?key|bearer|authorization)\s*[:=]\s*\S+`)
privacyUsrAt = regexp.MustCompile(`[A-Za-z0-9._-]+@[A-Za-z0-9.-]+`)
)
// privacyGlyphs is a small set of dense unicode shade characters. By
// cycling through them the redacted range looks like diffused noise
// rather than a flat mask.
var privacyGlyphs = []rune{'▚', '▞', '▓', '▒'}
// privacyMask returns a redaction string the same visual length as src.
func privacyMask(src string) string {
rs := []rune(src)
out := make([]rune, len(rs))
for i, r := range rs {
if r == ' ' || r == '\t' || r == '\n' {
out[i] = r
continue
}
out[i] = privacyGlyphs[i%len(privacyGlyphs)]
}
return string(out)
}
// privacyRedact scrubs every sensitive pattern from the given line. The
// function is intentionally line-scoped — callers apply it row by row so
// that multi-line ANSI layouts survive the substitution.
func privacyRedact(line string) string {
if line == "" {
return line
}
replace := func(re *regexp.Regexp, s string) string {
return re.ReplaceAllStringFunc(s, privacyMask)
}
// Order matters: scrub the longest/most specific patterns first so
// later passes don't hit already-masked text.
line = privacyKey.ReplaceAllStringFunc(line, func(m string) string {
// Keep the label (password/token/etc) but mask the value.
idx := strings.IndexAny(m, "=:")
if idx < 0 {
return privacyMask(m)
}
return m[:idx+1] + privacyMask(strings.TrimLeft(m[idx+1:], " "))
})
line = replace(privacyB64, line)
line = replace(privacyHex, line)
line = replace(privacyMAC, line)
line = replace(privacyIPv4, line)
line = replace(privacyIPv6, line)
line = replace(privacyUsrAt, line)
line = replace(privacyHost, line)
return line
}
// applyPrivacyMultiline redacts every line independently. Safe to call
// on ANSI-styled output — masks only the literal runs a regex matches.
func applyPrivacyMultiline(s string) string {
if s == "" {
return s
}
lines := strings.Split(s, "\n")
for i, ln := range lines {
lines[i] = privacyRedact(ln)
}
return strings.Join(lines, "\n")
}
internal/cli/monitor_ssh.go
+466
View File
@@ -0,0 +1,466 @@
package cli
import (
"context"
"fmt"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/hinshun/vt10x"
"pxmon/internal/cluster"
)
// sshStartedMsg is dispatched when an embedded SSH session finishes dialing.
type sshStartedMsg struct {
session *cluster.InteractiveSession
cluster cluster.Cluster
cols int
rows int
err error
}
// sshChunkMsg delivers a chunk of remote PTY output to the model.
type sshChunkMsg struct {
data []byte
err error
}
// sshClosedMsg signals that the embedded session was torn down.
type sshClosedMsg struct {
err error
}
// sshTickMsg throttles vt10x → view repaints while output is flowing.
type sshTickMsg struct{}
const (
sshMinCols = 20
sshMinRows = 5
sshChunkBuffer = 16384
sshRepaintInterval = 33 * time.Millisecond
)
// sshEmbeddedDims returns the usable grid size for the SSH panel.
func (m monitorModel) sshEmbeddedDims() (int, int) {
w := m.width
if w <= 0 {
w = 120
}
h := m.height
if h <= 0 {
h = 32
}
cols := w - 4
rows := h - 4
if cols < sshMinCols {
cols = sshMinCols
}
if rows < sshMinRows {
rows = sshMinRows
}
return cols, rows
}
// startEmbeddedSSHCmd kicks off an SSH dial in a goroutine and returns
// the started session through an sshStartedMsg.
func (m monitorModel) startEmbeddedSSHCmd(selector string) tea.Cmd {
svc := m.svc
cols, rows := m.sshEmbeddedDims()
c, err := svc.Get(selector)
if err != nil {
return func() tea.Msg { return sshStartedMsg{err: err} }
}
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
sess, err := svc.StartInteractiveShell(ctx, c.ID, cols, rows, "xterm-256color")
if err != nil {
return sshStartedMsg{err: err, cluster: c}
}
return sshStartedMsg{
session: sess,
cluster: c,
cols: cols,
rows: rows,
}
}
}
// readSSHChunkCmd reads the next chunk from the SSH session in a goroutine.
func readSSHChunkCmd(sess *cluster.InteractiveSession) tea.Cmd {
if sess == nil {
return nil
}
return func() tea.Msg {
buf := make([]byte, sshChunkBuffer)
n, err := sess.Read(buf)
if n > 0 {
data := make([]byte, n)
copy(data, buf[:n])
return sshChunkMsg{data: data, err: err}
}
return sshChunkMsg{err: err}
}
}
// sshRepaintTickCmd schedules a throttled repaint while output flows.
func sshRepaintTickCmd() tea.Cmd {
return tea.Tick(sshRepaintInterval, func(_ time.Time) tea.Msg {
return sshTickMsg{}
})
}
// enterEmbeddedSSH switches the model into embedded SSH view for the
// freshly dialed session.
func (m *monitorModel) enterEmbeddedSSH(msg sshStartedMsg) {
cols := msg.cols
rows := msg.rows
if cols < sshMinCols {
cols = sshMinCols
}
if rows < sshMinRows {
rows = sshMinRows
}
vt := vt10x.New(vt10x.WithSize(cols, rows))
m.sshMode = true
m.sshSess = msg.session
m.sshCluster = msg.cluster
m.sshVT = vt
m.sshCols = cols
m.sshRows = rows
m.sshClosed = false
m.sshErr = ""
m.sshPendingRepaint = false
}
// exitEmbeddedSSH tears down the embedded session and clears state.
func (m *monitorModel) exitEmbeddedSSH(reason string) {
if m.sshSess != nil {
_ = m.sshSess.Close()
}
m.sshSess = nil
m.sshVT = nil
m.sshMode = false
m.sshClosed = true
m.sshPendingRepaint = false
// Return to pxmon console so the user lands where they launched from.
m.termMode = true
m.termFull = false
if reason != "" {
m.setStatus(reason)
}
}
// writeSSHInput pushes a byte slice to the SSH session's stdin.
func (m monitorModel) writeSSHInput(p []byte) {
if m.sshSess == nil || len(p) == 0 {
return
}
_, _ = m.sshSess.Write(p)
}
// handleSSHKey routes TUI key events into the remote PTY stdin.
func (m monitorModel) handleSSHKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
// Escape hatch: Ctrl+] closes the embedded session.
if v.Type == tea.KeyCtrlCloseBracket {
m.exitEmbeddedSSH(fmt.Sprintf("ssh closed (%s)", m.sshCluster.Name))
m.appendConsoleOutput(fmt.Sprintf("ssh session closed (%s)", m.sshCluster.Name))
return m, nil
}
// Scrollback controls — mirror the pxmon console (alt+↑/↓, pgup/pgdn,
// alt+home/end). Alt+Shift+↑/↓ jump 10 lines at a time for fast review.
// These never reach the remote PTY.
switch v.String() {
case "alt+up":
m.scrollSSH(1)
return m, nil
case "alt+down":
m.scrollSSH(-1)
return m, nil
case "alt+shift+up":
m.scrollSSH(10)
return m, nil
case "alt+shift+down":
m.scrollSSH(-10)
return m, nil
case "pgup":
m.scrollSSH(m.sshRows - 1)
return m, nil
case "pgdown":
m.scrollSSH(-(m.sshRows - 1))
return m, nil
case "alt+home":
m.sshScrollOffset = len(m.sshScrollback)
m.clampSSHScroll()
return m, nil
case "alt+end":
m.sshScrollOffset = 0
return m, nil
case "alt+p":
m.privacyMode = !m.privacyMode
if m.privacyMode {
m.setStatus("privacy: on")
} else {
m.setStatus("privacy: off")
}
return m, nil
}
// Any other input resumes live view if the user was scrolling back.
if m.sshScrollOffset != 0 {
m.sshScrollOffset = 0
}
payload := keyMsgToPTY(v)
if len(payload) == 0 {
return m, nil
}
m.writeSSHInput(payload)
return m, nil
}
// captureSSHScrollback appends remote output (with ANSI sequences stripped)
// to the scrollback buffer so the user can scroll through past output even
// though vt10x does not retain a scroll history of its own.
func (m *monitorModel) captureSSHScrollback(data []byte) {
if len(data) == 0 {
return
}
text := stripANSI(string(data))
// Treat stand-alone CR as a rewrite of the current line; drop content
// before the CR to avoid duplicating progress-bar style updates.
combined := m.sshScrollPending + text
combined = strings.ReplaceAll(combined, "\r\n", "\n")
lines := strings.Split(combined, "\n")
// Last element is either a trailing newline ("") or a partial line; stash.
m.sshScrollPending = lines[len(lines)-1]
lines = lines[:len(lines)-1]
for _, ln := range lines {
if idx := strings.LastIndex(ln, "\r"); idx >= 0 {
ln = ln[idx+1:]
}
m.sshScrollback = append(m.sshScrollback, ln)
}
const maxScrollback = 5000
if len(m.sshScrollback) > maxScrollback {
m.sshScrollback = m.sshScrollback[len(m.sshScrollback)-maxScrollback:]
}
// A new chunk means more history arrived behind the current scroll
// window — adjust offset so the user keeps looking at the same line.
if m.sshScrollOffset > 0 {
m.sshScrollOffset += len(lines)
m.clampSSHScroll()
}
}
func (m *monitorModel) scrollSSH(delta int) {
m.sshScrollOffset += delta
m.clampSSHScroll()
}
func (m *monitorModel) clampSSHScroll() {
maxOffset := len(m.sshScrollback) - m.sshRows
if maxOffset < 0 {
maxOffset = 0
}
if m.sshScrollOffset > maxOffset {
m.sshScrollOffset = maxOffset
}
if m.sshScrollOffset < 0 {
m.sshScrollOffset = 0
}
}
// handleSSHResize adjusts the vt10x grid and notifies the remote side.
func (m *monitorModel) handleSSHResize() {
if !m.sshMode || m.sshVT == nil || m.sshSess == nil {
return
}
cols, rows := m.sshEmbeddedDims()
if cols == m.sshCols && rows == m.sshRows {
return
}
m.sshCols = cols
m.sshRows = rows
m.sshVT.Resize(cols, rows)
_ = m.sshSess.Resize(cols, rows)
}
// renderSSHView produces the TUI view for the embedded SSH session.
func (m monitorModel) renderSSHView() string {
header := accentStyle.Bold(true).Render(fmt.Sprintf(" ssh://%s@%s:%d (%s) ", m.sshCluster.User, m.sshCluster.Host, m.sshCluster.Port, m.sshCluster.Name))
hint := dimStyle.Render("Ctrl+] detach · alt+↑/↓ scroll · pgup/pgdn page · alt+end live")
if m.sshScrollOffset > 0 {
hint = warnStyle.Render(fmt.Sprintf("↑ scrolled +%d (alt+end to follow)", m.sshScrollOffset)) +
dimStyle.Render(" Ctrl+] detach")
}
body := m.renderVTBody()
panel := lipgloss.NewStyle().
BorderStyle(thinBorder).
BorderForeground(ccAccent).
Padding(0, 1).
Render(body)
status := dimStyle.Render(strings.TrimSpace("status: " + m.statusMsg))
return lipgloss.JoinVertical(lipgloss.Left, header+" "+hint, panel, status)
}
// renderVTBody iterates the vt10x grid and emits a styled string block.
// When the user has scrolled back it instead renders a window of the
// ANSI-stripped scrollback buffer.
func (m monitorModel) renderVTBody() string {
if m.sshVT == nil {
return dimStyle.Render("(session not ready)")
}
if m.sshScrollOffset > 0 {
return m.renderScrollbackBody()
}
vt := m.sshVT
vt.Lock()
cols, rows := vt.Size()
cur := vt.Cursor()
cursorVisible := vt.CursorVisible()
var lines []string
for y := 0; y < rows; y++ {
line := renderVTRow(vt, y, cols, cursorVisible, cur.X, cur.Y)
lines = append(lines, line)
}
vt.Unlock()
return strings.Join(lines, "\n")
}
// renderScrollbackBody paints a page of the captured scrollback buffer when
// the user is browsing history. Every row is padded to the full grid width
// so the panel keeps its live dimensions — otherwise lipgloss sizes the
// border to the longest line and the view visibly collapses.
func (m monitorModel) renderScrollbackBody() string {
rows := m.sshRows
cols := m.sshCols
if rows <= 0 {
rows = sshMinRows
}
if cols <= 0 {
cols = sshMinCols
}
end := len(m.sshScrollback) - m.sshScrollOffset
if end < 0 {
end = 0
}
start := end - rows
if start < 0 {
start = 0
}
blank := strings.Repeat(" ", cols)
out := make([]string, 0, rows)
pad := func(line string) string {
rs := []rune(line)
if len(rs) > cols {
rs = rs[:cols]
}
if len(rs) < cols {
return string(rs) + strings.Repeat(" ", cols-len(rs))
}
return string(rs)
}
for i := start; i < end; i++ {
out = append(out, pad(m.sshScrollback[i]))
}
for len(out) < rows {
out = append(out, blank)
}
return strings.Join(out, "\n")
}
// renderVTRow renders a single grid row as a styled string.
func renderVTRow(vt vt10x.Terminal, row, cols int, cursorVisible bool, cursorX, cursorY int) string {
var b strings.Builder
var (
runRunes strings.Builder
runFG vt10x.Color = vt10x.DefaultFG
runBG vt10x.Color = vt10x.DefaultBG
runStart = true
)
flush := func() {
if runRunes.Len() == 0 {
return
}
style := styleForColors(runFG, runBG)
b.WriteString(style.Render(runRunes.String()))
runRunes.Reset()
}
for x := 0; x < cols; x++ {
cell := vt.Cell(x, row)
ch := cell.Char
if ch == 0 {
ch = ' '
}
fg := cell.FG
bg := cell.BG
isCursor := cursorVisible && x == cursorX && row == cursorY
if runStart {
runFG = fg
runBG = bg
runStart = false
}
if isCursor {
// Emit the current run first, then paint the cursor cell with
// an explicit, always-visible color so the caret shows even on
// empty cells with default FG/BG (where a plain invert would
// collapse to the same color).
flush()
cursorStyle := lipgloss.NewStyle().
Background(ccAccent).
Foreground(lipgloss.Color("#101010")).
Bold(true)
b.WriteString(cursorStyle.Render(string(ch)))
runFG = fg
runBG = bg
continue
}
if fg != runFG || bg != runBG {
flush()
runFG = fg
runBG = bg
}
runRunes.WriteRune(ch)
}
flush()
return b.String()
}
// styleForColors returns a lipgloss style for the given vt10x colors.
func styleForColors(fg, bg vt10x.Color) lipgloss.Style {
style := lipgloss.NewStyle()
if c, ok := vtColorToLipgloss(fg); ok {
style = style.Foreground(c)
}
if c, ok := vtColorToLipgloss(bg); ok {
style = style.Background(c)
}
return style
}
// vtColorToLipgloss maps a vt10x color to a lipgloss color. Returns ok=false
// for default so the caller leaves the attribute unset.
func vtColorToLipgloss(c vt10x.Color) (lipgloss.TerminalColor, bool) {
if c == vt10x.DefaultFG || c == vt10x.DefaultBG || c == vt10x.DefaultCursor {
return nil, false
}
// ANSI basic 16
if c < 16 {
return lipgloss.Color(fmt.Sprintf("%d", uint32(c))), true
}
// 256-color palette
if c < 256 {
return lipgloss.Color(fmt.Sprintf("%d", uint32(c))), true
}
// Truecolor (24-bit): stored in low 24 bits.
r := (uint32(c) >> 16) & 0xff
g := (uint32(c) >> 8) & 0xff
bl := uint32(c) & 0xff
return lipgloss.Color(fmt.Sprintf("#%02x%02x%02x", r, g, bl)), true
}
internal/cli/monitor_usage.go
+241
View File
@@ -0,0 +1,241 @@
package cli
import (
"context"
"fmt"
"strings"
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"pxmon/internal/cluster"
"pxmon/internal/history"
)
type monitorUsageMsg struct {
snap cluster.UsageSnapshot
err error
}
func (m monitorModel) fetchUsageCmd(rng history.RangeShortcut) tea.Cmd {
svc := m.svc
selector := m.cluster.ID
duPath := "/"
return func() tea.Msg {
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, duPath)
return monitorUsageMsg{snap: snap, err: err}
}
}
func (m monitorModel) handleUsageKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
switch v.String() {
case "o", "esc":
m.view = viewOverview
return m, nil
case "n":
m.view = viewNetwork
return m, nil
case "c":
m.view = viewClusters
return m, nil
case "s":
m.view = viewSettings
return m, nil
case "r", "R":
m.usageLoading = true
m.setStatus("usage: refreshing")
return m, m.fetchUsageCmd(m.usageRange)
case "1":
m.usageRange = history.RangeLive
m.usageLoading = true
m.setStatus("usage: live")
return m, m.fetchUsageCmd(m.usageRange)
case "2":
m.usageRange = history.RangeHour
m.usageLoading = true
m.setStatus("usage: last 1h")
return m, m.fetchUsageCmd(m.usageRange)
case "3":
m.usageRange = history.RangeDay
m.usageLoading = true
m.setStatus("usage: last 24h")
return m, m.fetchUsageCmd(m.usageRange)
case "4":
m.usageRange = history.RangeMonth
m.usageLoading = true
m.setStatus("usage: last 30d")
return m, m.fetchUsageCmd(m.usageRange)
case "5":
m.usageRange = history.RangeAll
m.usageLoading = true
m.setStatus("usage: all time")
return m, m.fetchUsageCmd(m.usageRange)
}
return m, nil
}
func (m monitorModel) renderUsageDashboard(width int) string {
title := titleStyle.Render("━━ Usage / billing ━━")
help := dimStyle.Render("range: 1=live 2=1h 3=1d 4=1mo 5=all r refresh o back")
if m.usageLoading && m.usageSnap == nil {
body := dimStyle.Render("loading usage snapshot…")
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
}
if m.usageSnap == nil {
if m.usageErr != nil {
body := critStyle.Render("error: " + m.usageErr.Error())
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
}
body := dimStyle.Render("press u to load usage")
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
}
snap := *m.usageSnap
rng := m.usageRange
lines := []string{title, help, ""}
if m.usageLoading {
lines = append(lines, dimStyle.Render("(refreshing…)"))
}
if m.usageErr != nil {
lines = append(lines, critStyle.Render("error: "+m.usageErr.Error()))
}
lines = append(lines,
fmt.Sprintf("%s %s %s %s %s %s %s %d",
dimStyle.Render("cluster:"), accentStyle.Render(snap.ClusterName),
dimStyle.Render("range:"), brightStyle.Render(rng.Label()),
dimStyle.Render("cpu:"), brightStyle.Render(fmt.Sprintf("%.1f%%", snap.Live.CPU.UsagePercent)),
dimStyle.Render("procs:"), snap.Top.TotalProcs,
),
fmt.Sprintf("%s %s %s %s",
dimStyle.Render("ram:"), brightStyle.Render(fmt.Sprintf("%.1f%%", snap.Live.Memory.UsedPercent)),
dimStyle.Render("host:"), accentStyle.Render(snap.Live.Host.Hostname),
),
"",
titleStyle.Render("── Network (uplink, max(Rx,Tx)) ──"),
)
if snap.HistoryError != "" {
lines = append(lines, warnStyle.Render("history: "+snap.HistoryError))
}
lines = append(lines,
fmt.Sprintf(" %s %s %s %s %s %s (%d samples)",
dimStyle.Render("P95:"), okStyle.Render(formatMbpsHuman(snap.P95TotalMbps)),
dimStyle.Render("max:"), brightStyle.Render(formatMbpsHuman(snap.MaxTotalMbps)),
dimStyle.Render("avg:"), brightStyle.Render(formatMbpsHuman(snap.AvgTotalMbps)),
len(snap.NodeSeries),
),
)
if snap.TopIfaceName != "" {
lines = append(lines,
fmt.Sprintf(" %s %s (avg %s)",
dimStyle.Render("uplink:"),
accentStyle.Render(snap.TopIfaceName),
formatMbpsHuman(snap.TopIfaceMbps)),
)
}
// Sparkline from node series
if len(snap.NodeSeries) > 1 {
lines = append(lines, " "+dimStyle.Render("series:")+" "+renderNodeSparkline(snap.NodeSeries, 40))
}
lines = append(lines, "", titleStyle.Render("── Top processes (by CPU) ──"))
if snap.TopError != "" {
lines = append(lines, warnStyle.Render("top: "+snap.TopError))
} else {
for i, p := range snap.Top.TopByCPU {
if i >= 6 {
break
}
lines = append(lines, fmt.Sprintf(" %5d %-10s %s%% %8s %s",
p.PID,
clipRight(p.User, 10),
brightStyle.Render(fmt.Sprintf("%5.1f", p.CPUPercent)),
humanBytesUint(p.RSSBytes),
accentStyle.Render(clipRight(p.Command, 36))))
}
}
lines = append(lines, "", titleStyle.Render("── Top processes (by RAM) ──"))
if snap.TopError == "" {
for i, p := range snap.Top.TopByMemory {
if i >= 6 {
break
}
lines = append(lines, fmt.Sprintf(" %5d %-10s %8s %s",
p.PID,
clipRight(p.User, 10),
brightStyle.Render(humanBytesUint(p.RSSBytes)),
accentStyle.Render(clipRight(p.Command, 44))))
}
}
lines = append(lines, "", titleStyle.Render("── Top folders ──"))
if snap.DUError != "" {
lines = append(lines, warnStyle.Render("du: "+snap.DUError))
} else {
for i, d := range snap.DU.Entries {
if i >= 8 {
break
}
lines = append(lines, fmt.Sprintf(" %10s %s",
brightStyle.Render(humanBytesUint(d.Bytes)),
accentStyle.Render(d.Path)))
}
if snap.DU.Truncated {
lines = append(lines, warnStyle.Render(" (scan truncated by timeout)"))
}
}
return strings.Join(lines, "\n")
}
func renderNodeSparkline(series []history.NodeSamplePoint, width int) string {
if len(series) == 0 || width <= 0 {
return ""
}
maxV := 0.0
for _, p := range series {
if p.TotalMbps > maxV {
maxV = p.TotalMbps
}
}
if maxV <= 0 {
return strings.Repeat("·", width)
}
step := len(series) / width
if step < 1 {
step = 1
}
runes := []rune{' ', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'}
var b strings.Builder
for i := 0; i < len(series); i += step {
v := series[i].TotalMbps
idx := int((v / maxV) * float64(len(runes)-1))
if idx < 0 {
idx = 0
}
if idx >= len(runes) {
idx = len(runes) - 1
}
b.WriteRune(runes[idx])
}
return b.String()
}
func clipRight(s string, n int) string {
if len(s) <= n {
return s
}
if n <= 3 {
return s[:n]
}
return s[:n-3] + "..."
}
internal/cli/shell.go
+286
View File
@@ -0,0 +1,286 @@
package cli
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"strings"
"unicode"
"pxmon/internal/cluster"
)
type shellStatus struct {
ActiveCluster string
ClusterCount int
LoadError error
}
func (a *App) runShell(configPath string, jsonOut bool) int {
status := a.loadShellStatus(configPath)
a.printShellBanner(status, configPath, jsonOut)
history := make([]string, 0, 64)
reader := bufio.NewReader(os.Stdin)
for {
status = a.loadShellStatus(configPath)
fmt.Fprint(a.out, status.prompt())
line, err := reader.ReadString('\n')
if err != nil {
if errors.Is(err, io.EOF) {
fmt.Fprintln(a.out)
return 0
}
fmt.Fprintf(a.err, "shell input error: %v\n", err)
return 1
}
line = strings.TrimSpace(line)
if line == "" {
continue
}
history = append(history, line)
args, parseErr := parseShellArgs(line)
if parseErr != nil {
fmt.Fprintf(a.err, "parse error: %v\n", parseErr)
continue
}
if len(args) == 0 {
continue
}
if strings.EqualFold(args[0], "pxmon") || strings.EqualFold(args[0], "pxmon") {
args = args[1:]
}
if len(args) == 0 {
continue
}
switch shellToken(args[0]) {
case "exit", "quit", "q", ":q":
return 0
case "clear", "cls":
fmt.Fprint(a.out, "\x1b[2J\x1b[H")
continue
case "history":
for i, h := range history {
fmt.Fprintf(a.out, "%3d %s\n", i+1, h)
}
continue
case "help", "?":
a.printShellHelp()
continue
case "status", "dashboard":
a.printShellStatus(status, configPath, jsonOut)
continue
}
cmd := expandShellCommand(args)
code := a.runRootCommand(cmd, configPath, jsonOut, false)
if code != 0 {
fmt.Fprintf(a.err, "[exit %d]\n", code)
}
}
}
func (a *App) loadShellStatus(configPath string) shellStatus {
status := shellStatus{}
store, err := cluster.NewStore(configPath)
if err != nil {
status.LoadError = err
return status
}
svc := cluster.NewService(store)
clusters, _, err := svc.List()
if err != nil {
status.LoadError = err
return status
}
status.ClusterCount = len(clusters)
current, err := svc.Current()
if err == nil {
status.ActiveCluster = current.Name
return status
}
if !errors.Is(err, cluster.ErrNoActiveCluster) {
status.LoadError = err
}
return status
}
func (s shellStatus) prompt() string {
active := strings.TrimSpace(s.ActiveCluster)
if active == "" {
active = "none"
}
active = strings.ReplaceAll(active, "]", "_")
return fmt.Sprintf("pxmon[%s]> ", active)
}
func (a *App) printShellBanner(status shellStatus, configPath string, jsonOut bool) {
fmt.Fprintln(a.out, "+----------------------------------------------------------------+")
fmt.Fprintln(a.out, "| pxmon interactive shell |")
fmt.Fprintln(a.out, "| slash: /help /clusters /use <name> /stats [name] /network |")
fmt.Fprintln(a.out, "| exit: /quit or /q |")
fmt.Fprintln(a.out, "+----------------------------------------------------------------+")
a.printShellStatus(status, configPath, jsonOut)
fmt.Fprintln(a.out)
}
func (a *App) printShellStatus(status shellStatus, configPath string, jsonOut bool) {
active := strings.TrimSpace(status.ActiveCluster)
if active == "" {
active = "none"
}
jsonState := "off"
if jsonOut {
jsonState = "on"
}
fmt.Fprintf(a.out, "Clusters: %d | Active: %s | JSON: %s\n", status.ClusterCount, active, jsonState)
if strings.TrimSpace(configPath) != "" {
fmt.Fprintf(a.out, "Config: %s\n", configPath)
}
if status.LoadError != nil {
fmt.Fprintf(a.out, "Status error: %v\n", status.LoadError)
}
}
func (a *App) printShellHelp() {
fmt.Fprintln(a.out, "Shell commands:")
fmt.Fprintln(a.out, " help Show this help")
fmt.Fprintln(a.out, " status Show shell status")
fmt.Fprintln(a.out, " history Show command history")
fmt.Fprintln(a.out, " clear Clear screen")
fmt.Fprintln(a.out, " exit | quit | q Exit shell")
fmt.Fprintln(a.out)
fmt.Fprintln(a.out, "Shortcuts:")
fmt.Fprintln(a.out, " /clusters Alias for 'cluster list'")
fmt.Fprintln(a.out, " /stats [name] Alias for 'cluster stats [name]'")
fmt.Fprintln(a.out, " /monitor [name] Alias for 'cluster stats [name]'")
fmt.Fprintln(a.out, " /network [name] Open network-focused TUI")
fmt.Fprintln(a.out, " /connect ... Alias for 'cluster connect ...'")
fmt.Fprintln(a.out, " /use <name> Alias for 'cluster use <name>'")
fmt.Fprintln(a.out, " /alert ... Alias for 'cluster alert ...'")
fmt.Fprintln(a.out)
fmt.Fprintln(a.out, "Examples:")
fmt.Fprintln(a.out, " /clusters")
fmt.Fprintln(a.out, " /connect --name eu-1 --host 10.0.0.10 --user root --auth key --key-path ~/.ssh/id_ed25519")
fmt.Fprintln(a.out, " /bootstrap eu-1")
fmt.Fprintln(a.out, " /stats eu-1")
fmt.Fprintln(a.out, " /network eu-1")
fmt.Fprintln(a.out, " /alert set eu-1 --net-mbps 300 --ram 90 --disk 90")
fmt.Fprintln(a.out, " /alert set eu-1 --net-sustain-enabled=true --net-sustain-mbps 500 --net-sustain-mins 60 --net-sustain-include net0 --net-sustain-exclude backup")
}
func shellToken(v string) string {
token := strings.TrimSpace(strings.ToLower(v))
if strings.HasPrefix(token, "/") {
token = strings.TrimPrefix(token, "/")
}
return token
}
func expandShellCommand(args []string) []string {
if len(args) == 0 {
return args
}
out := append([]string(nil), args...)
cmd := shellToken(out[0])
if cmd == "" {
return out
}
out[0] = cmd
switch cmd {
case "clusters", "nodes":
return append([]string{"cluster", "list"}, out[1:]...)
case "stats", "monitor", "watch":
return append([]string{"cluster", "stats"}, out[1:]...)
case "alerts":
return append([]string{"cluster", "alert"}, out[1:]...)
}
if isClusterShortcut(cmd) {
return append([]string{"cluster", cmd}, out[1:]...)
}
return out
}
func isClusterShortcut(cmd string) bool {
switch cmd {
case "connect", "add", "list", "ls", "show", "get", "current", "use",
"ping", "check", "bootstrap", "disconnect", "remove", "rm", "alert":
return true
default:
return false
}
}
func parseShellArgs(line string) ([]string, error) {
line = strings.TrimSpace(line)
if line == "" {
return nil, nil
}
args := make([]string, 0, 8)
var current strings.Builder
var quote rune
escaped := false
tokenStarted := false
flush := func() {
if tokenStarted {
args = append(args, current.String())
current.Reset()
tokenStarted = false
}
}
for _, r := range line {
switch {
case escaped:
current.WriteRune(r)
escaped = false
tokenStarted = true
case r == '\\':
escaped = true
tokenStarted = true
case quote != 0:
if r == quote {
quote = 0
} else {
current.WriteRune(r)
}
tokenStarted = true
case r == '\'' || r == '"':
quote = r
tokenStarted = true
case unicode.IsSpace(r):
flush()
default:
current.WriteRune(r)
tokenStarted = true
}
}
if escaped {
return nil, errors.New("unterminated escape at end of command")
}
if quote != 0 {
return nil, errors.New("unterminated quoted string")
}
flush()
return args, nil
}
internal/cli/usage.go
+414
View File
@@ -0,0 +1,414 @@
package cli
import (
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"text/tabwriter"
"time"
"pxmon/internal/agent"
"pxmon/internal/cluster"
"pxmon/internal/history"
)
func (a *App) runClusterUsage(svc *cluster.Service, args []string, jsonOut bool) int {
fs := flag.NewFlagSet("cluster usage", flag.ContinueOnError)
fs.SetOutput(a.err)
rangeStr := fs.String("range", "live", "Time window: live|1h|1d|1mo|all")
duPath := fs.String("du", "/", "Root directory for top-folder scan")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
selector := ""
if fs.NArg() > 0 {
selector = fs.Arg(0)
}
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr))
if !ok {
fmt.Fprintf(a.err, "usage: invalid --range %q (expected live|1h|1d|1mo|all)\n", *rangeStr)
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, *duPath)
if err != nil {
fmt.Fprintf(a.err, "cluster usage: %v\n", err)
return 1
}
if jsonOut {
_ = writeJSON(a.out, snap)
return 0
}
printUsageSnapshot(a.out, snap, rng)
return 0
}
func (a *App) runClusterTraffic(svc *cluster.Service, args []string, jsonOut bool) int {
fs := flag.NewFlagSet("cluster traffic", flag.ContinueOnError)
fs.SetOutput(a.err)
rangeStr := fs.String("range", "1h", "Time window: 1h|1d|1mo|all")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
selector := ""
if fs.NArg() > 0 {
selector = fs.Arg(0)
}
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr))
if !ok {
fmt.Fprintf(a.err, "traffic: invalid --range %q\n", *rangeStr)
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, "")
if err != nil {
fmt.Fprintf(a.err, "cluster traffic: %v\n", err)
return 1
}
if jsonOut {
_ = writeJSON(a.out, map[string]any{
"cluster": snap.ClusterName,
"range": rng.Label(),
"p95_mbps": snap.P95TotalMbps,
"max_mbps": snap.MaxTotalMbps,
"avg_mbps": snap.AvgTotalMbps,
"samples": len(snap.NodeSeries),
"top_iface": snap.TopIfaceName,
"top_iface_mbps": snap.TopIfaceMbps,
"history_error": snap.HistoryError,
})
return 0
}
fmt.Fprintf(a.out, "%s %s %s\n", colorLabel("Cluster:"), colorAccent(snap.ClusterName), colorMuted("("+rng.Label()+")"))
if snap.HistoryError != "" {
fmt.Fprintf(a.out, "%s %s\n", colorWarn("History:"), snap.HistoryError)
}
fmt.Fprintf(a.out, "%s %s %s %d samples\n",
colorLabel("P95: "),
colorOK(formatMbpsHuman(snap.P95TotalMbps)),
colorMuted("over"),
len(snap.NodeSeries),
)
fmt.Fprintf(a.out, "%s %s %s %s\n",
colorLabel("Max: "),
colorValue(formatMbpsHuman(snap.MaxTotalMbps)),
colorMuted("avg:"),
colorInfo(formatMbpsHuman(snap.AvgTotalMbps)),
)
if snap.TopIfaceName != "" {
fmt.Fprintf(a.out, "%s %s %s %s\n",
colorLabel("Uplink:"),
colorAccent(snap.TopIfaceName),
colorMuted("avg max(Rx,Tx):"),
colorValue(formatMbpsHuman(snap.TopIfaceMbps)),
)
}
return 0
}
func (a *App) runClusterGraph(svc *cluster.Service, args []string, jsonOut bool) int {
fs := flag.NewFlagSet("cluster graph", flag.ContinueOnError)
fs.SetOutput(a.err)
rangeStr := fs.String("range", "1d", "Time window: 1h|1d|1mo|all")
outPath := fs.String("out", "", "Output PNG path (default: ./<cluster>-<range>.png)")
if err := fs.Parse(args); err != nil {
if errors.Is(err, flag.ErrHelp) {
return 0
}
return 2
}
selector := ""
if fs.NArg() > 0 {
selector = fs.Arg(0)
}
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(*rangeStr))
if !ok {
fmt.Fprintf(a.err, "graph: invalid --range %q\n", *rangeStr)
return 2
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
snap, err := svc.CollectUsageSnapshot(ctx, selector, rng, "")
if err != nil {
fmt.Fprintf(a.err, "cluster graph: %v\n", err)
return 1
}
png, err := cluster.RenderUsageChartPNG(snap, "")
if err != nil {
fmt.Fprintf(a.err, "cluster graph: render: %v\n", err)
return 1
}
dest := strings.TrimSpace(*outPath)
if dest == "" {
dest = filepath.Join(".", fmt.Sprintf("%s-%s.png", sanitizeFilename(snap.ClusterName), rng))
}
if err := os.WriteFile(dest, png, 0o600); err != nil {
fmt.Fprintf(a.err, "cluster graph: write: %v\n", err)
return 1
}
if jsonOut {
_ = writeJSON(a.out, map[string]any{
"path": dest,
"bytes": len(png),
"p95_mbps": snap.P95TotalMbps,
"samples": len(snap.NodeSeries),
})
return 0
}
fmt.Fprintf(a.out, "%s %s\n", colorLabel("Saved:"), colorAccent(dest))
fmt.Fprintf(a.out, "%s %s %s %d samples\n",
colorLabel("P95: "),
colorOK(formatMbpsHuman(snap.P95TotalMbps)),
colorMuted("over"),
len(snap.NodeSeries),
)
return 0
}
func printUsageSnapshot(w io.Writer, snap cluster.UsageSnapshot, rng history.RangeShortcut) {
fmt.Fprintf(w, "%s %s %s\n", colorLabel("Cluster:"), colorAccent(snap.ClusterName), colorMuted("("+rng.Label()+")"))
fmt.Fprintf(w, "%s %s %s\n", colorLabel("Host: "), colorAccent(snap.Live.Host.Hostname), colorMuted(snap.Live.Host.OS+"/"+snap.Live.Host.Arch))
fmt.Fprintf(w, "%s %s %s %s %s %s\n",
colorLabel("CPU:"),
colorValue(fmt.Sprintf("%.1f%%", snap.Live.CPU.UsagePercent)),
colorLabel("RAM:"),
colorValue(fmt.Sprintf("%.1f%%", snap.Live.Memory.UsedPercent)),
colorLabel("Procs:"),
colorInfo(fmt.Sprintf("%d", snap.Top.TotalProcs)),
)
fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Network ━━"))
if snap.HistoryError != "" {
fmt.Fprintf(w, "%s %s\n", colorWarn("history:"), snap.HistoryError)
}
fmt.Fprintf(w, " %s %s %s %s %s %s (%d samples)\n",
colorLabel("P95:"), colorOK(formatMbpsHuman(snap.P95TotalMbps)),
colorLabel("max:"), colorValue(formatMbpsHuman(snap.MaxTotalMbps)),
colorLabel("avg:"), colorInfo(formatMbpsHuman(snap.AvgTotalMbps)),
len(snap.NodeSeries),
)
if snap.TopIfaceName != "" {
fmt.Fprintf(w, " %s %s %s %s\n",
colorLabel("uplink:"),
colorAccent(snap.TopIfaceName),
colorMuted("avg max(Rx,Tx)"),
colorValue(formatMbpsHuman(snap.TopIfaceMbps)),
)
}
if snap.TopError != "" {
fmt.Fprintf(w, "\n%s %s\n", colorWarn("top:"), snap.TopError)
} else {
fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Top processes (by CPU) ━━"))
printProcessTable(w, snap.Top.TopByCPU)
fmt.Fprintf(w, "\n%s\n", colorHeader("━━ Top processes (by RAM) ━━"))
printProcessTable(w, snap.Top.TopByMemory)
}
if snap.DUError != "" {
fmt.Fprintf(w, "\n%s %s\n", colorWarn("du:"), snap.DUError)
} else {
fmt.Fprintf(w, "\n%s %s\n", colorHeader("━━ Top folders ━━"), colorMuted(snap.DU.Root))
printFolderTable(w, snap.DU.Entries)
if snap.DU.Truncated {
fmt.Fprintf(w, " %s\n", colorWarn("(scan truncated by timeout)"))
}
}
}
func printProcessTable(w io.Writer, procs []agent.ProcessStat) {
if len(procs) == 0 {
fmt.Fprintln(w, " (no data)")
return
}
tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0)
fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\n",
colorHeader("PID"), colorHeader("USER"), colorHeader("CPU%"), colorHeader("RSS"), colorHeader("CMD"))
for _, p := range procs {
if p.CPUPercent == 0 && p.RSSBytes == 0 {
continue
}
cmd := p.Command
if len(cmd) > 40 {
cmd = cmd[:37] + "..."
}
fmt.Fprintf(tw, " %s\t%s\t%s\t%s\t%s\n",
colorInfo(fmt.Sprintf("%d", p.PID)),
colorMuted(p.User),
colorValue(fmt.Sprintf("%.1f", p.CPUPercent)),
colorValue(humanBytesUint(p.RSSBytes)),
colorAccent(cmd),
)
}
_ = tw.Flush()
}
func printFolderTable(w io.Writer, dirs []agent.DirStat) {
if len(dirs) == 0 {
fmt.Fprintln(w, " (no data)")
return
}
tw := tabwriter.NewWriter(w, 0, 2, 2, ' ', 0)
fmt.Fprintf(tw, " %s\t%s\t%s\n",
colorHeader("SIZE"), colorHeader("FILES"), colorHeader("PATH"))
for _, d := range dirs {
fmt.Fprintf(tw, " %s\t%s\t%s\n",
colorValue(humanBytesUint(d.Bytes)),
colorInfo(fmt.Sprintf("%d", d.Files)),
colorAccent(d.Path),
)
}
_ = tw.Flush()
}
func formatMbpsHuman(v float64) string {
switch {
case v >= 1000:
return fmt.Sprintf("%.2f Gbps", v/1000)
case v >= 1:
return fmt.Sprintf("%.1f Mbps", v)
case v > 0:
return fmt.Sprintf("%.0f Kbps", v*1000)
default:
return "0 Mbps"
}
}
func humanBytesUint(v uint64) string {
const unit = 1024
if v < unit {
return fmt.Sprintf("%d B", v)
}
div, exp := uint64(unit), 0
for n := v / unit; n >= unit; n /= unit {
div *= unit
exp++
}
pre := "KMGTPE"
return fmt.Sprintf("%.2f %ciB", float64(v)/float64(div), pre[exp])
}
func sanitizeFilename(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return "cluster"
}
var b bytes.Buffer
for _, r := range name {
switch {
case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
return b.String()
}
// formatUsageForBot returns a plain-text rendering (no ANSI) suitable for
// wrapping in a <pre> block in Telegram.
func formatUsageForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string {
var b strings.Builder
fmt.Fprintf(&b, "Cluster: %s (%s)\n", snap.ClusterName, rng.Label())
fmt.Fprintf(&b, "Host: %s %s/%s\n", snap.Live.Host.Hostname, snap.Live.Host.OS, snap.Live.Host.Arch)
fmt.Fprintf(&b, "CPU: %5.1f%% RAM: %5.1f%% Procs: %d\n",
snap.Live.CPU.UsagePercent, snap.Live.Memory.UsedPercent, snap.Top.TotalProcs)
b.WriteString("\n── Network ──\n")
if snap.HistoryError != "" {
fmt.Fprintf(&b, "history: %s\n", snap.HistoryError)
}
fmt.Fprintf(&b, "P95 %s · max %s · avg %s (%d samples)\n",
formatMbpsHuman(snap.P95TotalMbps),
formatMbpsHuman(snap.MaxTotalMbps),
formatMbpsHuman(snap.AvgTotalMbps),
len(snap.NodeSeries),
)
if snap.TopIfaceName != "" {
fmt.Fprintf(&b, "uplink: %s (avg max(Rx,Tx): %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps))
}
b.WriteString("\n── Top by CPU ──\n")
writeBotProcLine(&b, snap.Top.TopByCPU, 5)
b.WriteString("── Top by RAM ──\n")
writeBotProcLine(&b, snap.Top.TopByMemory, 5)
if snap.DUError == "" {
fmt.Fprintf(&b, "\n── Top folders (%s) ──\n", snap.DU.Root)
for _, d := range snap.DU.Entries {
if len(snap.DU.Entries) > 8 {
break
}
fmt.Fprintf(&b, "%10s %s\n", humanBytesUint(d.Bytes), d.Path)
}
// if large, take first 8 regardless
n := len(snap.DU.Entries)
if n > 8 {
for i := 0; i < 8; i++ {
d := snap.DU.Entries[i]
fmt.Fprintf(&b, "%10s %s\n", humanBytesUint(d.Bytes), d.Path)
}
}
}
return b.String()
}
func writeBotProcLine(b *strings.Builder, procs []agent.ProcessStat, n int) {
if len(procs) == 0 {
b.WriteString("(no data)\n")
return
}
if n > len(procs) {
n = len(procs)
}
for i := 0; i < n; i++ {
p := procs[i]
cmd := p.Command
if len(cmd) > 28 {
cmd = cmd[:25] + "..."
}
user := p.User
if len(user) > 8 {
user = user[:8]
}
fmt.Fprintf(b, "%5d %-8s %5.1f%% %8s %s\n",
p.PID, user, p.CPUPercent, humanBytesUint(p.RSSBytes), cmd)
}
}
// writeJSON is used by subcommands for --json output. Declared in app.go.
var _ = json.Marshal
internal/cli/winch_unix.go
+41
View File
@@ -0,0 +1,41 @@
//go:build !windows
package cli
import (
"os"
"os/signal"
"syscall"
"golang.org/x/term"
"pxmon/internal/cluster"
)
func installWinchHandler(fd int, resize chan<- cluster.InteractiveShellSize) chan os.Signal {
sigCh := make(chan os.Signal, 4)
signal.Notify(sigCh, syscall.SIGWINCH)
go func() {
for range sigCh {
w, h, err := term.GetSize(fd)
if err != nil || w <= 0 || h <= 0 {
continue
}
select {
case resize <- cluster.InteractiveShellSize{Width: w, Height: h}:
default:
}
}
}()
return sigCh
}
func closeWinchHandler(sigCh chan os.Signal, resize chan cluster.InteractiveShellSize) {
if sigCh != nil {
signal.Stop(sigCh)
close(sigCh)
}
if resize != nil {
close(resize)
}
}
internal/cli/winch_windows.go
+19
View File
@@ -0,0 +1,19 @@
//go:build windows
package cli
import (
"os"
"pxmon/internal/cluster"
)
func installWinchHandler(fd int, resize chan<- cluster.InteractiveShellSize) chan os.Signal {
return nil
}
func closeWinchHandler(sigCh chan os.Signal, resize chan cluster.InteractiveShellSize) {
if resize != nil {
close(resize)
}
}
internal/cluster/agent_auth.go
+42
View File
@@ -0,0 +1,42 @@
package cluster
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"net/http"
"strconv"
"strings"
"time"
)
const (
agentHeaderTS = "X-Observer-Ts"
agentHeaderNonce = "X-Observer-Nonce"
agentHeaderSignature = "X-Observer-Signature"
)
func applyAgentRequestAuth(req *http.Request, c Cluster) {
if req == nil {
return
}
if strings.TrimSpace(c.Agent.Token) != "" {
req.Header.Set("Authorization", "Bearer "+c.Agent.Token)
}
secret := strings.TrimSpace(c.Agent.RequestSecret)
if secret == "" {
return
}
ts := strconv.FormatInt(time.Now().UTC().Unix(), 10)
nonce := randomHex(12)
payload := req.Method + "\n" + req.URL.RequestURI() + "\n" + ts + "\n" + nonce
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(payload))
sig := hex.EncodeToString(mac.Sum(nil))
req.Header.Set(agentHeaderTS, ts)
req.Header.Set(agentHeaderNonce, nonce)
req.Header.Set(agentHeaderSignature, sig)
}
internal/cluster/agent_versions.go
+68
View File
@@ -0,0 +1,68 @@
package cluster
import (
"embed"
"encoding/json"
"strings"
)
//go:embed agent_versions.json
var agentVersionsFS embed.FS
type AgentVersionInfo struct {
Version string `json:"version"`
ReleasedAt string `json:"released_at,omitempty"`
Features []string `json:"features,omitempty"`
}
func loadAgentVersions() []AgentVersionInfo {
raw, err := agentVersionsFS.ReadFile("agent_versions.json")
if err != nil {
return nil
}
var items []AgentVersionInfo
if err := json.Unmarshal(raw, &items); err != nil {
return nil
}
out := make([]AgentVersionInfo, 0, len(items))
for _, it := range items {
it.Version = strings.TrimSpace(it.Version)
if it.Version == "" {
continue
}
out = append(out, it)
}
return out
}
func (s *Service) AgentVersions() []AgentVersionInfo {
return append([]AgentVersionInfo(nil), loadAgentVersions()...)
}
func (s *Service) AgentVersionFeatures(version string) []string {
v := strings.TrimSpace(version)
if v == "" {
return nil
}
for _, it := range loadAgentVersions() {
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
return append([]string(nil), it.Features...)
}
}
return nil
}
func (s *Service) CompareAgentVersion(version string) (isLatest bool, latest string, known bool) {
items := loadAgentVersions()
if len(items) == 0 {
return true, "", false
}
latest = strings.TrimSpace(items[len(items)-1].Version)
v := strings.TrimSpace(version)
for _, it := range items {
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
return strings.EqualFold(v, latest), latest, true
}
}
return false, latest, false
}
internal/cluster/agent_versions.json
+37
View File
@@ -0,0 +1,37 @@
[
{
"version": "dev",
"released_at": "2026-04-10",
"features": [
"cluster usage/traffic/graph",
"kvm top static spec",
"telegram graph sendPhoto fallback"
]
},
{
"version": "dev-2026.04.17",
"released_at": "2026-04-17",
"features": [
"cluster p95 by interface",
"cluster tag + kvm tag",
"cluster vm alert-rules",
"cluster drift",
"cluster runbook",
"cluster scheduler",
"cluster change-history",
"cluster report export"
]
},
{
"version": "v0.2.0",
"released_at": "2026-06-16",
"features": [
"repo tunneling gateway/proxy workflows",
"cluster exec/run commands",
"ssh key passphrase file support",
"export/import referenced key files",
"signed agent request skew hardening",
"tui refresh and network dashboard improvements"
]
}
]
internal/cluster/backup.go
+655
View File
@@ -0,0 +1,655 @@
package cluster
import (
"context"
"fmt"
"io"
"net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type BackupRunResult struct {
PlanID string `json:"plan_id"`
PlanName string `json:"plan_name"`
TargetID string `json:"target_id"`
TargetName string `json:"target_name"`
ArchiveName string `json:"archive_name"`
UploadedTo string `json:"uploaded_to"`
SizeBytes int64 `json:"size_bytes"`
RanAt time.Time `json:"ran_at"`
}
func (s *Service) BackupListTargets() ([]BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil
}
func (s *Service) BackupListPlans() ([]BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil
}
func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) {
reg, err := s.store.Load()
if err != nil {
return "", err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
var target *BackupTarget
for i := range cfg.Targets {
if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return "", fmt.Errorf("backup target not found")
}
switch target.Type {
case "sftp":
host := strings.TrimSpace(target.SFTPHost)
user := strings.TrimSpace(target.SFTPUser)
if host == "" || user == "" {
return "", fmt.Errorf("sftp target has empty host/user")
}
addr := netJoinHostPort(host, target.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(target.SFTPPassword) != "" {
auths = append(auths, ssh.Password(target.SFTPPassword))
}
if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(target.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
if _, err := c.ReadDir(base); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil
case "s3":
endpoint := strings.TrimSpace(target.S3Endpoint)
bucket := strings.TrimSpace(target.S3Bucket)
access := strings.TrimSpace(target.S3AccessKey)
secret := strings.TrimSpace(target.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(target.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if target.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: target.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
exists, err := cli.BucketExists(ctx, bucket)
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket)
}
scheme := "https"
if !target.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket, nil
default:
return "", fmt.Errorf("unsupported target type %q", target.Type)
}
}
func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
t.ID = strings.TrimSpace(t.ID)
if t.ID == "" {
t.ID = newClusterID()
}
t.Name = strings.TrimSpace(t.Name)
if t.Name == "" {
return BackupTarget{}, fmt.Errorf("target name is required")
}
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
if t.Type != "sftp" && t.Type != "s3" {
return BackupTarget{}, fmt.Errorf("target type must be sftp|s3")
}
if t.Type == "sftp" {
if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user")
}
if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires password or key")
}
}
if t.Type == "s3" {
if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket")
}
if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key")
}
}
for _, ex := range cfg.Targets {
if strings.EqualFold(ex.Name, t.Name) {
return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name)
}
}
now := s.now().UTC()
t.CreatedAt = now
t.UpdatedAt = now
if t.SFTPPort <= 0 {
t.SFTPPort = 22
}
if !t.Enabled {
t.Enabled = true
}
cfg.Targets = append(cfg.Targets, t)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.add", t.ID, t.Name)
return t, nil
}
func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, t := range cfg.Targets {
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupTarget{}, fmt.Errorf("backup target not found")
}
removed := cfg.Targets[idx]
cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
if strings.TrimSpace(p.Name) == "" {
return BackupPlan{}, fmt.Errorf("plan name is required")
}
if strings.TrimSpace(p.TargetID) == "" {
return BackupPlan{}, fmt.Errorf("target is required")
}
if len(p.Paths) == 0 {
return BackupPlan{}, fmt.Errorf("at least one path is required")
}
targetID := ""
for _, t := range cfg.Targets {
if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) {
targetID = t.ID
break
}
}
if targetID == "" {
return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID)
}
for _, ex := range cfg.Plans {
if strings.EqualFold(ex.Name, p.Name) {
return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name)
}
}
p.ID = newClusterID()
p.TargetID = targetID
p.Paths = normalizeBackupPaths(p.Paths)
if strings.TrimSpace(p.Every) == "" {
p.Every = "24h"
}
if p.RetainDays <= 0 {
p.RetainDays = 30
}
p.Compress = true
now := s.now().UTC()
p.CreatedAt = now
p.UpdatedAt = now
if !p.Enabled {
p.Enabled = true
}
cfg.Plans = append(cfg.Plans, p)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.add", p.ID, p.Name)
return p, nil
}
func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, p := range cfg.Plans {
if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupPlan{}, fmt.Errorf("backup plan not found")
}
removed := cfg.Plans[idx]
cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) {
reg, err := s.store.Load()
if err != nil {
return BackupRunResult{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
var plan *BackupPlan
for i := range cfg.Plans {
if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) {
plan = &cfg.Plans[i]
break
}
}
if plan == nil {
return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector)
}
var target *BackupTarget
for i := range cfg.Targets {
if cfg.Targets[i].ID == plan.TargetID {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID)
}
c, err := s.Get(plan.Cluster)
if err != nil {
return BackupRunResult{}, err
}
sshClient, err := s.dialSSH(ctx, c, "", "")
if err != nil {
return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err)
}
defer sshClient.Close()
ts := s.now().UTC().Format("20060102T150405Z")
archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz"
tmpPath := filepath.Join(os.TempDir(), archiveName)
tmpFile, err := os.Create(tmpPath)
if err != nil {
return BackupRunResult{}, err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpPath)
}()
remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths)
if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err)
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return BackupRunResult{}, err
}
st, _ := tmpFile.Stat()
size := int64(0)
if st != nil {
size = st.Size()
}
uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size)
if err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("upload backup: %w", err)
}
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "ok"
plan.LastError = ""
plan.LastArchive = archiveName
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupRunResult{}, err
}
_ = s.AppendChange("backup.plan.run", plan.ID, archiveName)
return BackupRunResult{
PlanID: plan.ID,
PlanName: plan.Name,
TargetID: target.ID,
TargetName: target.Name,
ArchiveName: archiveName,
UploadedTo: uploadedTo,
SizeBytes: size,
RanAt: plan.LastRunAt,
}, nil
}
func normalizeBackupPaths(in []string) []string {
out := make([]string, 0, len(in))
seen := map[string]struct{}{}
for _, p := range in {
v := strings.TrimSpace(p)
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
return out
}
func sanitizeBackupName(v string) string {
v = strings.ToLower(strings.TrimSpace(v))
if v == "" {
return "backup"
}
var b strings.Builder
for _, r := range v {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
return "backup"
}
return out
}
func buildRemoteTarStreamCommand(paths []string) string {
items := make([]string, 0, len(paths))
for _, p := range paths {
v := strings.TrimSpace(p)
if v == "" {
continue
}
items = append(items, shellQuote(v))
}
if len(items) == 0 {
items = []string{shellQuote("/")}
}
return "tar -czf - " + strings.Join(items, " ")
}
func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error {
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stdout, err := session.StdoutPipe()
if err != nil {
return err
}
stderr, err := session.StderrPipe()
if err != nil {
return err
}
if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
return err
}
done := make(chan error, 1)
go func() {
_, cpErr := io.Copy(w, stdout)
if cpErr != nil {
done <- cpErr
return
}
done <- session.Wait()
}()
select {
case <-ctx.Done():
_ = session.Close()
return ctx.Err()
case err := <-done:
if err == nil {
return nil
}
b, _ := io.ReadAll(stderr)
msg := strings.TrimSpace(string(b))
if msg == "" {
return err
}
return fmt.Errorf("%w: %s", err, msg)
}
}
func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
switch strings.ToLower(strings.TrimSpace(target.Type)) {
case "sftp":
return uploadBackupSFTP(ctx, target, archiveName, r)
case "s3":
return uploadBackupS3(ctx, target, archiveName, r, size)
default:
return "", fmt.Errorf("unsupported backup target type %q", target.Type)
}
}
func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) {
_ = ctx
host := strings.TrimSpace(t.SFTPHost)
if host == "" {
return "", fmt.Errorf("sftp_host is required")
}
user := strings.TrimSpace(t.SFTPUser)
if user == "" {
return "", fmt.Errorf("sftp_user is required")
}
addr := netJoinHostPort(host, t.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(t.SFTPPassword) != "" {
auths = append(auths, ssh.Password(t.SFTPPassword))
}
if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(t.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
remote := path.Join(base, archiveName)
f, err := c.Create(remote)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(f, r); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil
}
func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
endpoint := strings.TrimSpace(t.S3Endpoint)
bucket := strings.TrimSpace(t.S3Bucket)
access := strings.TrimSpace(t.S3AccessKey)
secret := strings.TrimSpace(t.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(t.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if t.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: t.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
key := archiveName
if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" {
key = p + "/" + archiveName
}
opts := minio.PutObjectOptions{ContentType: "application/gzip"}
if size < 0 {
size = -1
}
_, err = cli.PutObject(ctx, bucket, key, r, size, opts)
if err != nil {
return "", err
}
scheme := "https"
if !t.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil
}
func netJoinHostPort(host string, port int) string {
p := port
if p <= 0 {
p = 22
}
return net.JoinHostPort(host, strconv.Itoa(p))
}
internal/cluster/change_history.go
+89
View File
@@ -0,0 +1,89 @@
package cluster
import (
"bufio"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
type ChangeRecord struct {
At time.Time `json:"at"`
Action string `json:"action"`
Target string `json:"target"`
Details string `json:"details,omitempty"`
}
func (s *Service) changeHistoryPath() string {
return filepath.Join(s.DataDir(), "history", "changes.log")
}
func (s *Service) AppendChange(action, target, details string) error {
if s == nil {
return nil
}
rec := ChangeRecord{
At: s.now().UTC(),
Action: strings.TrimSpace(action),
Target: strings.TrimSpace(target),
Details: strings.TrimSpace(details),
}
if rec.Action == "" {
return nil
}
path := s.changeHistoryPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer f.Close()
b, err := json.Marshal(rec)
if err != nil {
return err
}
_, err = f.Write(append(b, '\n'))
return err
}
func (s *Service) ListChanges(limit int) ([]ChangeRecord, error) {
path := s.changeHistoryPath()
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer f.Close()
capHint := 32
if limit > capHint {
capHint = limit
}
items := make([]ChangeRecord, 0, capHint)
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var rec ChangeRecord
if err := json.Unmarshal([]byte(line), &rec); err != nil {
continue
}
items = append(items, rec)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("scan change history: %w", err)
}
if limit > 0 && len(items) > limit {
items = items[len(items)-limit:]
}
return items, nil
}
internal/cluster/drift.go
+86
View File
@@ -0,0 +1,86 @@
package cluster
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
type DriftIssue struct {
Level string `json:"level"`
Kind string `json:"kind"`
Message string `json:"message"`
}
type DriftReport struct {
Cluster string `json:"cluster"`
Generated time.Time `json:"generated_at"`
Issues []DriftIssue `json:"issues,omitempty"`
}
func (s *Service) DetectDrift(ctx context.Context, selector string) (DriftReport, error) {
c, err := s.Get(selector)
if err != nil {
return DriftReport{}, err
}
rep := DriftReport{Cluster: c.Name, Generated: s.now().UTC()}
expected := strings.TrimSpace(s.ExpectedAgentVersion())
dctrl := normalizeDriftControl(c.Drift)
if !c.Agent.Installed {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent", Message: "agent is not installed"})
return rep, nil
}
pingCtx, cancel := context.WithTimeout(ctx, 2500*time.Millisecond)
ping, pingErr := s.PingAgent(pingCtx, c.ID)
cancel()
if pingErr != nil || !ping.Reachable || ping.StatusCode >= 400 {
rep.Issues = append(rep.Issues, DriftIssue{Level: "crit", Kind: "agent", Message: "agent is unreachable"})
} else {
nodeVersion := strings.TrimSpace(ping.Version)
if nodeVersion == "" {
nodeVersion = strings.TrimSpace(c.Agent.Version)
}
if expected != "" && nodeVersion != "" && nodeVersion != expected {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_version", Message: fmt.Sprintf("agent version mismatch: node=%s local=%s", nodeVersion, expected)})
}
if nodeVersion != "" {
if ok, latest, known := s.CompareAgentVersion(nodeVersion); known && !ok {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_outdated", Message: fmt.Sprintf("node version %s is older than latest known %s", nodeVersion, latest)})
}
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.AgentVersion) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.AgentVersion), nodeVersion) {
rep.Issues = append(rep.Issues, DriftIssue{
Level: "warn",
Kind: "baseline_agent_version",
Message: fmt.Sprintf("baseline agent version mismatch: baseline=%s live=%s", dctrl.Baseline.AgentVersion, nodeVersion),
})
}
}
}
softCtx, softCancel := context.WithTimeout(ctx, 7*time.Second)
fresh, softErr := s.probeSoftware(softCtx, c, "", "")
softCancel()
if softErr != nil {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software_probe", Message: softErr.Error()})
} else {
oldSet := strings.TrimSpace(c.Software.Summary())
newSet := strings.TrimSpace(fresh.Summary())
if oldSet != "" && oldSet != "-" && newSet != oldSet {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software", Message: fmt.Sprintf("software support changed: stored=%s live=%s", oldSet, newSet)})
}
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.Software) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.Software), newSet) {
rep.Issues = append(rep.Issues, DriftIssue{
Level: "warn",
Kind: "baseline_software",
Message: fmt.Sprintf("baseline software mismatch: baseline=%s live=%s", dctrl.Baseline.Software, newSet),
})
}
}
sort.Slice(rep.Issues, func(i, j int) bool { return rep.Issues[i].Kind < rep.Issues[j].Kind })
return rep, nil
}
internal/cluster/export.go
+456
View File
@@ -0,0 +1,456 @@
package cluster
import (
"crypto/aes"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/crypto/scrypt"
)
const (
exportMagic = "OBSCTLEXP1:"
exportVersion = 2
saltBytes = 16
scryptN = 1 << 15
scryptR = 8
scryptP = 1
)
// ExportBundle is the portable payload produced by Service.Export. It is
// self-contained: it includes every cluster (with stored credentials),
// telegram bot settings, locker config, and alert policies.
type ExportBundle struct {
Version int `json:"version"`
ExportedAt string `json:"exported_at"`
Registry Registry `json:"registry"`
Files []ExportedFile `json:"files,omitempty"`
}
type ExportedFile struct {
OriginalPath string `json:"original_path"`
Kind string `json:"kind"`
Content []byte `json:"content"`
}
// ImportMode controls how an imported bundle is merged into the current store.
type ImportMode string
const (
// ImportModeMerge adds clusters from the bundle; existing clusters with
// the same name are replaced with the imported copy.
ImportModeMerge ImportMode = "merge"
// ImportModeReplace wipes the current registry and replaces it with the
// imported bundle as-is.
ImportModeReplace ImportMode = "replace"
)
// ImportReport summarizes what happened during an import.
type ImportReport struct {
Added int
Replaced int
TotalAfter int
TelegramApplied bool
LockerApplied bool
Mode ImportMode
}
// Export writes an encrypted, passphrase-protected bundle of the full
// registry to outPath. The bundle can be imported on another machine with
// the same password — no master key transfer required.
func (s *Service) Export(outPath, password string) error {
if strings.TrimSpace(outPath) == "" {
return errors.New("export: output path is required")
}
if strings.TrimSpace(password) == "" {
return errors.New("export: password is required")
}
reg, err := s.store.Load()
if err != nil {
return fmt.Errorf("export: load registry: %w", err)
}
files, err := collectExportFiles(reg)
if err != nil {
return err
}
bundle := ExportBundle{
Version: exportVersion,
ExportedAt: s.now().UTC().Format(time.RFC3339),
Registry: reg,
Files: files,
}
payload, err := json.MarshalIndent(bundle, "", " ")
if err != nil {
return fmt.Errorf("export: encode bundle: %w", err)
}
salt := make([]byte, saltBytes)
if _, err := rand.Read(salt); err != nil {
return fmt.Errorf("export: generate salt: %w", err)
}
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
if err != nil {
return fmt.Errorf("export: derive key: %w", err)
}
ciphertext, err := encrypt(payload, key)
if err != nil {
return fmt.Errorf("export: encrypt: %w", err)
}
blob := append([]byte{}, salt...)
blob = append(blob, ciphertext...)
encoded := exportMagic + base64.StdEncoding.EncodeToString(blob) + "\n"
if dir := filepath.Dir(outPath); dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0o700); err != nil {
return fmt.Errorf("export: create output dir: %w", err)
}
}
tmp := outPath + ".tmp"
if err := os.WriteFile(tmp, []byte(encoded), 0o600); err != nil {
return fmt.Errorf("export: write temp: %w", err)
}
if err := os.Rename(tmp, outPath); err != nil {
return fmt.Errorf("export: replace output: %w", err)
}
return nil
}
// Import reads an exported bundle from inPath using password, and applies
// it to the local registry according to mode.
func (s *Service) Import(inPath, password string, mode ImportMode) (ImportReport, error) {
if strings.TrimSpace(inPath) == "" {
return ImportReport{}, errors.New("import: input path is required")
}
if strings.TrimSpace(password) == "" {
return ImportReport{}, errors.New("import: password is required")
}
if mode == "" {
mode = ImportModeMerge
}
if mode != ImportModeMerge && mode != ImportModeReplace {
return ImportReport{}, fmt.Errorf("import: unsupported mode %q", mode)
}
raw, err := os.ReadFile(inPath)
if err != nil {
return ImportReport{}, fmt.Errorf("import: read input: %w", err)
}
text := strings.TrimSpace(string(raw))
if !strings.HasPrefix(text, exportMagic) {
return ImportReport{}, errors.New("import: not a PXmon export bundle")
}
blob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(text, exportMagic))
if err != nil {
return ImportReport{}, fmt.Errorf("import: decode payload: %w", err)
}
if len(blob) < saltBytes+aes.BlockSize {
return ImportReport{}, errors.New("import: payload truncated")
}
salt := blob[:saltBytes]
ciphertext := blob[saltBytes:]
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
if err != nil {
return ImportReport{}, fmt.Errorf("import: derive key: %w", err)
}
plaintext, err := decrypt(ciphertext, key)
if err != nil {
return ImportReport{}, errors.New("import: wrong password or corrupt bundle")
}
var bundle ExportBundle
if err := json.Unmarshal(plaintext, &bundle); err != nil {
return ImportReport{}, fmt.Errorf("import: parse bundle: %w", err)
}
if bundle.Version == 0 || bundle.Version > exportVersion {
return ImportReport{}, fmt.Errorf("import: unsupported bundle version %d", bundle.Version)
}
incoming := bundle.Registry
if incoming.Clusters == nil {
incoming.Clusters = []Cluster{}
}
if len(bundle.Files) > 0 {
restored, restoreErr := s.restoreExportFiles(bundle.Files)
if restoreErr != nil {
return ImportReport{}, restoreErr
}
applyRestoredFilePaths(&incoming, restored)
}
report := ImportReport{Mode: mode}
if mode == ImportModeReplace {
report.Added = len(incoming.Clusters)
report.TotalAfter = len(incoming.Clusters)
report.TelegramApplied = incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0
report.LockerApplied = incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled
if err := s.store.Save(incoming); err != nil {
return ImportReport{}, fmt.Errorf("import: save: %w", err)
}
return report, nil
}
current, err := s.store.Load()
if err != nil {
return ImportReport{}, fmt.Errorf("import: load current: %w", err)
}
if current.Clusters == nil {
current.Clusters = []Cluster{}
}
existingByName := make(map[string]int, len(current.Clusters))
for i, c := range current.Clusters {
existingByName[strings.ToLower(c.Name)] = i
}
for _, inc := range incoming.Clusters {
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key == "" {
continue
}
inc.Alerts = ensureAlertPolicy(inc.Alerts)
if idx, ok := existingByName[key]; ok {
// Preserve original ID to keep references stable.
inc.ID = current.Clusters[idx].ID
current.Clusters[idx] = inc
report.Replaced++
} else {
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
current.Clusters = append(current.Clusters, inc)
existingByName[key] = len(current.Clusters) - 1
report.Added++
}
}
if incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0 {
current.Telegram = incoming.Telegram
report.TelegramApplied = true
}
if incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled {
current.Locker = incoming.Locker
report.LockerApplied = true
}
current.Backups = mergeBackupConfig(current.Backups, incoming.Backups)
if strings.TrimSpace(current.ActiveClusterID) == "" {
current.ActiveClusterID = incoming.ActiveClusterID
}
report.TotalAfter = len(current.Clusters)
if err := s.store.Save(current); err != nil {
return ImportReport{}, fmt.Errorf("import: save: %w", err)
}
return report, nil
}
func collectExportFiles(reg Registry) ([]ExportedFile, error) {
type wantFile struct {
path string
kind string
}
wants := make([]wantFile, 0)
for _, c := range reg.Clusters {
if strings.TrimSpace(c.KeyPath) != "" {
wants = append(wants, wantFile{path: c.KeyPath, kind: "ssh_private_key"})
}
if strings.TrimSpace(c.KeyPassphraseFile) != "" {
wants = append(wants, wantFile{path: c.KeyPassphraseFile, kind: "ssh_key_passphrase"})
}
}
for _, t := range reg.Backups.Targets {
if strings.TrimSpace(t.SFTPKeyPath) != "" {
wants = append(wants, wantFile{path: t.SFTPKeyPath, kind: "sftp_private_key"})
}
}
seen := map[string]struct{}{}
files := make([]ExportedFile, 0, len(wants))
for _, w := range wants {
expanded, err := expandPath(w.path)
if err != nil {
return nil, fmt.Errorf("export: resolve %s file %q: %w", w.kind, w.path, err)
}
if _, ok := seen[expanded]; ok {
continue
}
data, err := os.ReadFile(expanded)
if err != nil {
return nil, fmt.Errorf("export: read %s file %q: %w", w.kind, expanded, err)
}
seen[expanded] = struct{}{}
files = append(files, ExportedFile{
OriginalPath: expanded,
Kind: w.kind,
Content: data,
})
}
return files, nil
}
func (s *Service) restoreExportFiles(files []ExportedFile) (map[string]string, error) {
base := filepath.Join(filepath.Dir(s.store.Path()), "imported-files")
if err := os.MkdirAll(base, 0o700); err != nil {
return nil, fmt.Errorf("import: create imported-files dir: %w", err)
}
restored := make(map[string]string, len(files))
for _, f := range files {
orig := strings.TrimSpace(f.OriginalPath)
if orig == "" {
continue
}
name := filepath.Base(orig)
if name == "." || name == string(filepath.Separator) || strings.TrimSpace(name) == "" {
name = "secret"
}
name = sanitizeExportFilename(name)
sum := sha256.Sum256([]byte(orig))
outPath := filepath.Join(base, hex.EncodeToString(sum[:6])+"-"+name)
if err := os.WriteFile(outPath, f.Content, 0o600); err != nil {
return nil, fmt.Errorf("import: restore file %q: %w", orig, err)
}
restored[orig] = outPath
}
return restored, nil
}
func applyRestoredFilePaths(reg *Registry, restored map[string]string) {
if reg == nil || len(restored) == 0 {
return
}
lookup := func(path string) string {
expanded, err := expandPath(path)
if err == nil {
if restoredPath := strings.TrimSpace(restored[expanded]); restoredPath != "" {
return restoredPath
}
}
if restoredPath := strings.TrimSpace(restored[strings.TrimSpace(path)]); restoredPath != "" {
return restoredPath
}
return path
}
for i := range reg.Clusters {
if strings.TrimSpace(reg.Clusters[i].KeyPath) != "" {
reg.Clusters[i].KeyPath = lookup(reg.Clusters[i].KeyPath)
}
if strings.TrimSpace(reg.Clusters[i].KeyPassphraseFile) != "" {
reg.Clusters[i].KeyPassphraseFile = lookup(reg.Clusters[i].KeyPassphraseFile)
}
}
for i := range reg.Backups.Targets {
if strings.TrimSpace(reg.Backups.Targets[i].SFTPKeyPath) != "" {
reg.Backups.Targets[i].SFTPKeyPath = lookup(reg.Backups.Targets[i].SFTPKeyPath)
}
}
}
func sanitizeExportFilename(name string) string {
var b strings.Builder
for _, r := range name {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '.', r == '_', r == '-':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
out := strings.Trim(b.String(), ".")
if out == "" {
return "secret"
}
return out
}
func mergeBackupConfig(current, incoming BackupConfig) BackupConfig {
if len(incoming.Targets) == 0 && len(incoming.Plans) == 0 {
return current
}
if current.Targets == nil {
current.Targets = []BackupTarget{}
}
if current.Plans == nil {
current.Plans = []BackupPlan{}
}
targetIDMap := map[string]string{}
targetByName := map[string]int{}
for i, t := range current.Targets {
if key := strings.ToLower(strings.TrimSpace(t.Name)); key != "" {
targetByName[key] = i
}
}
for _, inc := range incoming.Targets {
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key != "" {
if idx, ok := targetByName[key]; ok {
oldID := current.Targets[idx].ID
targetIDMap[inc.ID] = oldID
inc.ID = oldID
current.Targets[idx] = inc
continue
}
}
current.Targets = append(current.Targets, inc)
if key != "" {
targetByName[key] = len(current.Targets) - 1
}
}
planByName := map[string]int{}
for i, p := range current.Plans {
if key := strings.ToLower(strings.TrimSpace(p.Name)); key != "" {
planByName[key] = i
}
}
for _, inc := range incoming.Plans {
if mapped := strings.TrimSpace(targetIDMap[inc.TargetID]); mapped != "" {
inc.TargetID = mapped
}
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key != "" {
if idx, ok := planByName[key]; ok {
inc.ID = current.Plans[idx].ID
current.Plans[idx] = inc
continue
}
}
current.Plans = append(current.Plans, inc)
if key != "" {
planByName[key] = len(current.Plans) - 1
}
}
return current
}
internal/cluster/model.go
+518
View File
@@ -0,0 +1,518 @@
package cluster
import (
"sort"
"strings"
"time"
)
const currentVersion = 8
// AuthMethod describes how SSH authentication is performed.
type AuthMethod string
const (
AuthMethodPassword AuthMethod = "password"
AuthMethodKey AuthMethod = "key"
)
// TransportMode selects how pxmon reaches the agent HTTP API on a node.
type TransportMode string
const (
// TransportDirect is the default: pxmon dials the agent's listen
// address over plain TCP from the local machine.
TransportDirect TransportMode = "direct"
// TransportIPFabric tunnels the agent HTTP call through the existing SSH
// connection. Meant for nodes on ipfabric-style networking where the node
// has no default outbound route and we must not touch its network config.
// The agent is expected to bind to 127.0.0.1 on the node.
TransportIPFabric TransportMode = "ipfabric"
)
func normalizeTransport(t TransportMode) TransportMode {
switch strings.ToLower(strings.TrimSpace(string(t))) {
case "ipfabric", "ip-fabric", "ip_fabric":
return TransportIPFabric
case "", "direct":
return TransportDirect
default:
return TransportDirect
}
}
// Registry is the local inventory of managed clusters (nodes).
type Registry struct {
Version int `json:"version"`
ActiveClusterID string `json:"active_cluster_id,omitempty"`
Telegram Telegram `json:"telegram,omitempty"`
Locker Locker `json:"locker,omitempty"`
Backups BackupConfig `json:"backups,omitempty"`
Clusters []Cluster `json:"clusters"`
}
type BackupConfig struct {
Targets []BackupTarget `json:"targets,omitempty"`
Plans []BackupPlan `json:"plans,omitempty"`
}
type BackupTarget struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"` // sftp|s3
Enabled bool `json:"enabled"`
SFTPHost string `json:"sftp_host,omitempty"`
SFTPPort int `json:"sftp_port,omitempty"`
SFTPUser string `json:"sftp_user,omitempty"`
SFTPPassword string `json:"sftp_password,omitempty"`
SFTPKeyPath string `json:"sftp_key_path,omitempty"`
SFTPBasePath string `json:"sftp_base_path,omitempty"`
S3Endpoint string `json:"s3_endpoint,omitempty"`
S3Region string `json:"s3_region,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3Prefix string `json:"s3_prefix,omitempty"`
S3AccessKey string `json:"s3_access_key,omitempty"`
S3SecretKey string `json:"s3_secret_key,omitempty"`
S3UseSSL bool `json:"s3_use_ssl,omitempty"`
S3PathStyle bool `json:"s3_path_style,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type BackupPlan struct {
ID string `json:"id"`
Name string `json:"name"`
Cluster string `json:"cluster,omitempty"`
TargetID string `json:"target_id"`
Paths []string `json:"paths"`
Every string `json:"every,omitempty"`
Enabled bool `json:"enabled"`
RetainDays int `json:"retain_days,omitempty"`
Compress bool `json:"compress"`
LastRunAt time.Time `json:"last_run_at,omitempty"`
LastStatus string `json:"last_status,omitempty"`
LastArchive string `json:"last_archive,omitempty"`
LastError string `json:"last_error,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// Telegram stores Telegram bot integration settings.
type Telegram struct {
Enabled bool `json:"enabled,omitempty"`
Token string `json:"token,omitempty"`
AllowedUserIDs []int64 `json:"allowed_user_ids,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
// Locker stores global UI/CLI lock settings.
type Locker struct {
Enabled bool `json:"enabled,omitempty"`
PasswordHash string `json:"password_hash,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
// Cluster describes one managed node that we can reach over SSH.
type Cluster struct {
ID string `json:"id"`
Name string `json:"name"`
Host string `json:"host"`
Port int `json:"port"`
User string `json:"user"`
Transport TransportMode `json:"transport,omitempty"`
AuthMethod AuthMethod `json:"auth_method"`
Password string `json:"password,omitempty"`
KeyPath string `json:"key_path,omitempty"`
KeyPassphrase string `json:"key_passphrase,omitempty"`
KeyPassphraseFile string `json:"key_passphrase_file,omitempty"`
InsecureHostKey bool `json:"insecure_host_key,omitempty"`
Alerts AlertPolicy `json:"alerts"`
VMAlerts VMAlertPolicy `json:"vm_alerts,omitempty"`
AlertRouting AlertRoutingPolicy `json:"alert_routing,omitempty"`
RepoTunnel RepoTunnelState `json:"repo_tunnel,omitempty"`
RunbookTrigger RunbookTrigger `json:"runbook_trigger,omitempty"`
Drift DriftControl `json:"drift,omitempty"`
Tags []string `json:"tags,omitempty"`
KVMTags map[string][]string `json:"kvm_tags,omitempty"`
Agent AgentInstall `json:"agent,omitempty"`
Software SoftwareInfo `json:"software,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// AlertPolicy defines warning thresholds used by CLI monitor mode.
type AlertPolicy struct {
CPUWarnPercent float64 `json:"cpu_warn_percent"`
RAMWarnPercent float64 `json:"ram_warn_percent"`
SwapWarnPercent float64 `json:"swap_warn_percent"`
DiskWarnPercent float64 `json:"disk_warn_percent"`
NetWarnMbps float64 `json:"net_warn_mbps"`
NetSustainEnabled bool `json:"net_sustain_enabled,omitempty"`
NetSustainIface string `json:"net_sustain_iface,omitempty"`
NetSustainInclude []string `json:"net_sustain_include,omitempty"`
NetSustainExclude []string `json:"net_sustain_exclude,omitempty"`
NetSustainMbps float64 `json:"net_sustain_mbps,omitempty"`
NetSustainMinutes int `json:"net_sustain_minutes,omitempty"`
NetSustainCooldownMins int `json:"net_sustain_cooldown_mins,omitempty"`
}
// VMAlertPolicy controls KVM VM-state alerting per cluster.
type VMAlertPolicy struct {
Enabled bool `json:"enabled,omitempty"`
WarnOnShutoff bool `json:"warn_on_shutoff,omitempty"`
MinRunning int `json:"min_running,omitempty"`
}
// AlertRoutingPolicy controls delivery behavior for alerts (e.g. Telegram).
type AlertRoutingPolicy struct {
CriticalImmediate bool `json:"critical_immediate,omitempty"`
WarningBatchMins int `json:"warning_batch_mins,omitempty"`
}
// RunbookTrigger controls automatic runbook execution on selected events.
type RunbookTrigger struct {
Enabled bool `json:"enabled,omitempty"`
OnVMShutoff bool `json:"on_vm_shutoff,omitempty"`
RunbookID string `json:"runbook_id,omitempty"`
CooldownMins int `json:"cooldown_mins,omitempty"`
LastTriggered time.Time `json:"last_triggered,omitempty"`
}
// DriftBaseline stores expected values for drift comparisons.
type DriftBaseline struct {
Enabled bool `json:"enabled,omitempty"`
SetAt time.Time `json:"set_at,omitempty"`
AgentVersion string `json:"agent_version,omitempty"`
Software string `json:"software,omitempty"`
}
// DriftControl stores baseline and per-issue acknowledgement windows.
type DriftControl struct {
Baseline DriftBaseline `json:"baseline,omitempty"`
AckUntil map[string]time.Time `json:"ack_until,omitempty"`
}
// AgentInstall describes remote pxmon-agent installation details.
type AgentInstall struct {
Installed bool `json:"installed"`
Version string `json:"version,omitempty"`
RemoteBinary string `json:"remote_binary,omitempty"`
RemoteConfig string `json:"remote_config,omitempty"`
RemoteLog string `json:"remote_log,omitempty"`
RemotePIDFile string `json:"remote_pid_file,omitempty"`
ListenAddress string `json:"listen_address,omitempty"`
Port int `json:"port,omitempty"`
Token string `json:"token,omitempty"`
RequestSecret string `json:"request_secret,omitempty"`
TLSEnabled bool `json:"tls_enabled,omitempty"`
TLSCertPath string `json:"tls_cert_path,omitempty"`
TLSKeyPath string `json:"tls_key_path,omitempty"`
TLSFingerprint string `json:"tls_fingerprint,omitempty"`
LastBootstrapAt time.Time `json:"last_bootstrap_at,omitempty"`
}
// SoftwareInfo describes discovered software/plugins on a node.
type SoftwareInfo struct {
DetectedAt time.Time `json:"detected_at,omitempty"`
Bird bool `json:"bird,omitempty"`
FRR bool `json:"frr,omitempty"`
KVM bool `json:"kvm,omitempty"`
LXC bool `json:"lxc,omitempty"`
LXD bool `json:"lxd,omitempty"`
Versions map[string]string `json:"versions,omitempty"`
}
func (s SoftwareInfo) SupportedList() []string {
out := make([]string, 0, 5)
if s.Bird {
out = append(out, "bird")
}
if s.FRR {
out = append(out, "frr")
}
if s.KVM {
out = append(out, "kvm")
}
if s.LXC {
out = append(out, "lxc")
}
if s.LXD {
out = append(out, "lxd")
}
sort.Strings(out)
return out
}
func (s SoftwareInfo) Summary() string {
list := s.SupportedList()
if len(list) == 0 {
if !s.DetectedAt.IsZero() {
return "none"
}
return "-"
}
return strings.Join(list, ",")
}
func newRegistry() Registry {
return Registry{
Version: currentVersion,
Backups: normalizeBackupConfig(BackupConfig{}),
Clusters: []Cluster{},
}
}
func defaultAlertPolicy() AlertPolicy {
return AlertPolicy{
CPUWarnPercent: 85,
RAMWarnPercent: 90,
SwapWarnPercent: 80,
DiskWarnPercent: 90,
NetWarnMbps: 300,
}
}
func defaultVMAlertPolicy() VMAlertPolicy {
return VMAlertPolicy{
Enabled: false,
WarnOnShutoff: true,
MinRunning: 1,
}
}
func defaultAlertRoutingPolicy() AlertRoutingPolicy {
return AlertRoutingPolicy{
CriticalImmediate: true,
WarningBatchMins: 5,
}
}
func defaultRunbookTrigger() RunbookTrigger {
return RunbookTrigger{
Enabled: false,
OnVMShutoff: true,
CooldownMins: 30,
}
}
func ensureAlertPolicy(p AlertPolicy) AlertPolicy {
d := defaultAlertPolicy()
if p.CPUWarnPercent <= 0 {
p.CPUWarnPercent = d.CPUWarnPercent
}
if p.RAMWarnPercent <= 0 {
p.RAMWarnPercent = d.RAMWarnPercent
}
if p.SwapWarnPercent <= 0 {
p.SwapWarnPercent = d.SwapWarnPercent
}
if p.DiskWarnPercent <= 0 {
p.DiskWarnPercent = d.DiskWarnPercent
}
if p.NetWarnMbps <= 0 {
p.NetWarnMbps = d.NetWarnMbps
}
if p.NetSustainEnabled {
if p.NetSustainMbps <= 0 {
p.NetSustainMbps = d.NetWarnMbps
}
if p.NetSustainMinutes <= 0 {
p.NetSustainMinutes = 60
}
if p.NetSustainCooldownMins <= 0 {
p.NetSustainCooldownMins = 30
}
}
return p
}
func ensureVMAlertPolicy(p VMAlertPolicy) VMAlertPolicy {
d := defaultVMAlertPolicy()
wasZero := p == (VMAlertPolicy{})
if p.MinRunning <= 0 {
p.MinRunning = d.MinRunning
}
if !p.WarnOnShutoff {
// Keep explicit false if user set it, but default to true for zero-value
// policy loaded from old configs.
if wasZero {
p.WarnOnShutoff = d.WarnOnShutoff
}
}
return p
}
func ensureAlertRoutingPolicy(p AlertRoutingPolicy) AlertRoutingPolicy {
d := defaultAlertRoutingPolicy()
if p.WarningBatchMins <= 0 {
p.WarningBatchMins = d.WarningBatchMins
}
// default true when unset
if !p.CriticalImmediate && p == (AlertRoutingPolicy{}) {
p.CriticalImmediate = d.CriticalImmediate
}
return p
}
func ensureRunbookTrigger(p RunbookTrigger) RunbookTrigger {
d := defaultRunbookTrigger()
if p.CooldownMins <= 0 {
p.CooldownMins = d.CooldownMins
}
if !p.OnVMShutoff && p == (RunbookTrigger{}) {
p.OnVMShutoff = d.OnVMShutoff
}
p.RunbookID = strings.TrimSpace(p.RunbookID)
return p
}
func normalizeBackupConfig(cfg BackupConfig) BackupConfig {
if cfg.Targets == nil {
cfg.Targets = []BackupTarget{}
}
if cfg.Plans == nil {
cfg.Plans = []BackupPlan{}
}
for i := range cfg.Targets {
t := &cfg.Targets[i]
t.ID = strings.TrimSpace(t.ID)
t.Name = strings.TrimSpace(t.Name)
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
if t.SFTPPort <= 0 {
t.SFTPPort = 22
}
if t.ID == "" {
t.ID = newClusterID()
}
if t.Name == "" {
t.Name = t.ID
}
if t.Type != "sftp" && t.Type != "s3" {
t.Type = "sftp"
}
if !t.Enabled && t.CreatedAt.IsZero() {
t.Enabled = true
}
if t.CreatedAt.IsZero() {
t.CreatedAt = time.Now().UTC()
}
if t.UpdatedAt.IsZero() {
t.UpdatedAt = t.CreatedAt
}
}
for i := range cfg.Plans {
p := &cfg.Plans[i]
p.ID = strings.TrimSpace(p.ID)
p.Name = strings.TrimSpace(p.Name)
p.Cluster = strings.TrimSpace(p.Cluster)
p.TargetID = strings.TrimSpace(p.TargetID)
if p.ID == "" {
p.ID = newClusterID()
}
if p.Name == "" {
p.Name = p.ID
}
if p.Paths == nil {
p.Paths = []string{}
}
if p.Every == "" {
p.Every = "24h"
}
if p.RetainDays <= 0 {
p.RetainDays = 30
}
if !p.Compress {
p.Compress = true
}
if !p.Enabled && p.CreatedAt.IsZero() {
p.Enabled = true
}
if p.CreatedAt.IsZero() {
p.CreatedAt = time.Now().UTC()
}
if p.UpdatedAt.IsZero() {
p.UpdatedAt = p.CreatedAt
}
}
return cfg
}
func normalizeDriftControl(d DriftControl) DriftControl {
if len(d.AckUntil) == 0 {
d.AckUntil = nil
return d
}
out := make(map[string]time.Time, len(d.AckUntil))
for k, v := range d.AckUntil {
n := strings.ToLower(strings.TrimSpace(k))
if n == "" || v.IsZero() {
continue
}
out[n] = v.UTC()
}
if len(out) == 0 {
d.AckUntil = nil
} else {
d.AckUntil = out
}
d.Baseline.AgentVersion = strings.TrimSpace(d.Baseline.AgentVersion)
d.Baseline.Software = strings.TrimSpace(d.Baseline.Software)
return d
}
func normalizeTagList(tags []string) []string {
if len(tags) == 0 {
return nil
}
seen := make(map[string]struct{}, len(tags))
out := make([]string, 0, len(tags))
for _, t := range tags {
n := strings.ToLower(strings.TrimSpace(t))
if n == "" {
continue
}
if _, ok := seen[n]; ok {
continue
}
seen[n] = struct{}{}
out = append(out, n)
}
if len(out) == 0 {
return nil
}
sort.Strings(out)
return out
}
func normalizeVMTagMap(m map[string][]string) map[string][]string {
if len(m) == 0 {
return nil
}
out := make(map[string][]string, len(m))
for vm, tags := range m {
vmName := strings.TrimSpace(vm)
if vmName == "" {
continue
}
norm := normalizeTagList(tags)
if len(norm) == 0 {
continue
}
out[vmName] = norm
}
if len(out) == 0 {
return nil
}
return out
}
func normalizeLocker(cfg Locker) Locker {
cfg.PasswordHash = strings.TrimSpace(cfg.PasswordHash)
if cfg.PasswordHash == "" {
cfg.Enabled = false
}
return cfg
}
internal/cluster/p95.go
+82
View File
@@ -0,0 +1,82 @@
package cluster
import (
"errors"
"fmt"
"math"
"strings"
"pxmon/internal/history"
)
type InterfaceP95Snapshot struct {
ClusterName string `json:"cluster"`
ClusterID string `json:"cluster_id"`
Interface string `json:"interface"`
Range history.RangeShortcut `json:"range"`
Samples int `json:"samples"`
P95Mbps float64 `json:"p95_mbps"`
AvgMbps float64 `json:"avg_mbps"`
MaxMbps float64 `json:"max_mbps"`
Series []history.NodeSamplePoint `json:"series,omitempty"`
}
func (s *Service) CollectInterfaceP95(selector, iface string, rng history.RangeShortcut) (InterfaceP95Snapshot, error) {
iface = strings.TrimSpace(iface)
if iface == "" {
return InterfaceP95Snapshot{}, errors.New("interface is required")
}
c, err := s.Get(selector)
if err != nil {
return InterfaceP95Snapshot{}, err
}
store := s.NetworkStore()
if store == nil {
return InterfaceP95Snapshot{}, errors.New("history store not configured")
}
snaps, err := store.Load(c.ID, rng.Since(s.now()))
if err != nil {
return InterfaceP95Snapshot{}, err
}
series := history.AggregateNodeSeries(snaps, iface)
out := InterfaceP95Snapshot{
ClusterName: c.Name,
ClusterID: c.ID,
Interface: iface,
Range: rng,
Samples: len(series),
P95Mbps: history.PercentileMbps(series, 95),
Series: series,
}
if len(series) == 0 {
return out, nil
}
var sum, maxV float64
for _, p := range series {
sum += p.TotalMbps
if p.TotalMbps > maxV {
maxV = p.TotalMbps
}
}
out.AvgMbps = sum / float64(len(series))
out.MaxMbps = maxV
return out, nil
}
func (s *Service) RenderInterfaceP95GraphPNG(snap InterfaceP95Snapshot) ([]byte, error) {
if len(snap.Series) == 0 {
return history.RenderNodeNetworkPNG([]history.NodeSamplePoint{}, history.ChartOptions{
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
Subtitle: "No samples",
})
}
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps", snap.P95Mbps, snap.MaxMbps, snap.AvgMbps)
if math.IsNaN(snap.P95Mbps) {
subtitle = "No samples"
}
return history.RenderNodeNetworkPNG(snap.Series, history.ChartOptions{
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
Subtitle: subtitle,
Percentile: 95,
})
}
internal/cluster/policies.go
+152
View File
@@ -0,0 +1,152 @@
package cluster
import (
"errors"
"strings"
"time"
)
func (s *Service) GetAlertRouting(selector string) (AlertRoutingPolicy, error) {
c, err := s.Get(selector)
if err != nil {
return AlertRoutingPolicy{}, err
}
return ensureAlertRoutingPolicy(c.AlertRouting), nil
}
func (s *Service) SetAlertRouting(selector string, p AlertRoutingPolicy) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.AlertRouting = ensureAlertRoutingPolicy(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("alert.routing", c.Name, "updated")
return c, nil
}
func (s *Service) GetRunbookTrigger(selector string) (RunbookTrigger, error) {
c, err := s.Get(selector)
if err != nil {
return RunbookTrigger{}, err
}
return ensureRunbookTrigger(c.RunbookTrigger), nil
}
func (s *Service) SetRunbookTrigger(selector string, p RunbookTrigger) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.RunbookTrigger = ensureRunbookTrigger(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("runbook.trigger", c.Name, "updated")
return c, nil
}
func (s *Service) TouchRunbookTrigger(selector string, when time.Time) error {
reg, err := s.store.Load()
if err != nil {
return err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return err
}
tr := ensureRunbookTrigger(c.RunbookTrigger)
tr.LastTriggered = when.UTC()
c.RunbookTrigger = tr
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
return s.store.Save(reg)
}
func (s *Service) SetDriftBaseline(selector string) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
d := normalizeDriftControl(c.Drift)
d.Baseline = DriftBaseline{
Enabled: true,
SetAt: s.now().UTC(),
AgentVersion: strings.TrimSpace(c.Agent.Version),
Software: strings.TrimSpace(c.Software.Summary()),
}
c.Drift = d
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("drift.baseline", c.Name, "set")
return c, nil
}
func (s *Service) GetDriftControl(selector string) (DriftControl, error) {
c, err := s.Get(selector)
if err != nil {
return DriftControl{}, err
}
return normalizeDriftControl(c.Drift), nil
}
func (s *Service) AckDriftIssue(selector, issueKind string, until time.Time) (Cluster, error) {
kind := strings.ToLower(strings.TrimSpace(issueKind))
if kind == "" {
return Cluster{}, errors.New("issue kind is required")
}
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
d := normalizeDriftControl(c.Drift)
if d.AckUntil == nil {
d.AckUntil = map[string]time.Time{}
}
d.AckUntil[kind] = until.UTC()
c.Drift = d
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("drift.ack", c.Name, kind+" until="+until.UTC().Format(time.RFC3339))
return c, nil
}
func (s *Service) IsDriftIssueAcked(c Cluster, kind string, now time.Time) bool {
d := normalizeDriftControl(c.Drift)
if len(d.AckUntil) == 0 {
return false
}
u, ok := d.AckUntil[strings.ToLower(strings.TrimSpace(kind))]
if !ok {
return false
}
return now.UTC().Before(u)
}
internal/cluster/repo_tunnel.go
+413
View File
@@ -0,0 +1,413 @@
package cluster
import (
"context"
"errors"
"fmt"
"net"
"strconv"
"strings"
)
type RepoTunnelOptions struct {
Gateway string
GatewayIP string
Table int
Priority int
PackageManager string
Command string
KeepEnabled bool
NoRule bool
}
type RepoTunnelState struct {
Enabled bool `json:"enabled"`
Proxy string `json:"proxy,omitempty"`
Source string `json:"source,omitempty"`
}
func (s *Service) RepoTunnelEnable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
script, err := repoTunnelEnableScript(opts)
if err != nil {
return "", err
}
out, err := s.RunRemoteShell(ctx, selector, script)
if err != nil {
return "", err
}
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err == nil {
_ = s.updateRepoTunnelState(selector, RepoTunnelState{
Enabled: true,
Proxy: gw.proxyURL,
Source: strings.ToLower(strings.TrimSpace(opts.PackageManager)),
})
}
return out, nil
}
func (s *Service) RepoTunnelDisable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
script, err := repoTunnelDisableScript(opts)
if err != nil {
return "", err
}
out, err := s.RunRemoteShell(ctx, selector, script)
if err != nil {
return "", err
}
_ = s.updateRepoTunnelState(selector, RepoTunnelState{})
return out, nil
}
func (s *Service) RepoTunnelState(ctx context.Context, selector string) (RepoTunnelState, error) {
out, err := s.RunRemoteShell(ctx, selector, repoTunnelDetectScript())
if err != nil {
return RepoTunnelState{}, err
}
state := RepoTunnelState{}
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "proxy="):
state.Proxy = strings.TrimSpace(strings.TrimPrefix(line, "proxy="))
case strings.HasPrefix(line, "source="):
state.Source = strings.TrimSpace(strings.TrimPrefix(line, "source="))
}
}
state.Enabled = state.Proxy != ""
return state, nil
}
func (s *Service) updateRepoTunnelState(selector string, state RepoTunnelState) error {
reg, err := s.store.Load()
if err != nil {
return err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return err
}
c.RepoTunnel = state
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
return s.store.Save(reg)
}
func (s *Service) RepoTunnelStatus(ctx context.Context, selector string) (string, error) {
return s.RunRemoteShell(ctx, selector, repoTunnelStatusScript())
}
func (s *Service) RepoTunnelInstall(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
if strings.TrimSpace(opts.Command) == "" {
return "", errors.New("install command is required")
}
enableScript, err := repoTunnelEnableScript(opts)
if err != nil {
return "", err
}
disableScript, err := repoTunnelDisableScript(opts)
if err != nil {
return "", err
}
body := enableScript + "\n" + strings.TrimSpace(opts.Command) + "\n"
if !opts.KeepEnabled {
body = enableScript + "\ncleanup_pxmon_repo_tunnel() {\n" + disableScript + "\n}\ntrap cleanup_pxmon_repo_tunnel EXIT\n" + strings.TrimSpace(opts.Command) + "\n"
}
return s.RunRemoteShell(ctx, selector, body)
}
func RepoTunnelGatewayScript(port int, allowCIDRs []string) (string, error) {
if port == 0 {
port = 3128
}
if port < 1 || port > 65535 {
return "", errors.New("--port must be in range 1..65535")
}
if len(allowCIDRs) == 0 {
return "", errors.New("at least one --allow CIDR/IP is required")
}
aclParts := make([]string, 0, len(allowCIDRs))
for _, raw := range allowCIDRs {
v := strings.TrimSpace(raw)
if v == "" {
continue
}
if !validSquidSrcACL(v) {
return "", fmt.Errorf("invalid --allow %q; use an IP or CIDR without spaces", raw)
}
aclParts = append(aclParts, v)
}
if len(aclParts) == 0 {
return "", errors.New("at least one --allow CIDR/IP is required")
}
return fmt.Sprintf(`set -eu
if command -v dnf >/dev/null 2>&1; then
dnf install -y squid
elif command -v yum >/dev/null 2>&1; then
yum install -y squid
elif command -v apt-get >/dev/null 2>&1; then
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y squid
else
echo "no supported package manager found for squid install" >&2
exit 1
fi
conf=/etc/squid/squid.conf
cp -a "$conf" "$conf.pxmon-bak.$(date +%%Y%%m%%d%%H%%M%%S)"
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
block=$(mktemp)
{
echo "# BEGIN PXMON REPO TUNNEL"
if ! grep -Eq "^http_port[[:space:]]+([^[:space:]]+:)?%d\b" "$conf"; then
echo "http_port %d"
fi
echo "acl pxmon_repo_tunnel src %s"
echo "http_access allow pxmon_repo_tunnel"
echo "# END PXMON REPO TUNNEL"
} > "$block"
if grep -q "^http_access deny all" "$conf"; then
awk -v block="$block" '
BEGIN {while ((getline line < block) > 0) b = b line "\n"; close(block); inserted=0}
/^http_access deny all/ && !inserted {printf "%%s", b; inserted=1}
{print}
END {if (!inserted) printf "%%s", b}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
else
cat "$block" >> "$conf"
fi
rm -f "$block"
systemctl enable --now squid
systemctl restart squid
if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
firewall-cmd --add-port=%d/tcp --permanent
firewall-cmd --reload
fi
echo "pxmon repo gateway ready on port %d"
`, port, port, strings.Join(aclParts, " "), port, port), nil
}
func validSquidSrcACL(v string) bool {
for _, r := range v {
if r >= 'a' && r <= 'z' {
continue
}
if r >= 'A' && r <= 'Z' {
continue
}
if r >= '0' && r <= '9' {
continue
}
switch r {
case '.', ':', '/', '_', '-':
continue
default:
return false
}
}
return v != ""
}
func repoTunnelEnableScript(opts RepoTunnelOptions) (string, error) {
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err != nil {
return "", err
}
if !opts.NoRule && opts.Table <= 0 {
return "", errors.New("--table is required unless --no-rule is used")
}
manager := strings.ToLower(strings.TrimSpace(opts.PackageManager))
if manager == "" {
manager = "auto"
}
if manager != "auto" && manager != "apt" && manager != "dnf" && manager != "yum" {
return "", fmt.Errorf("unsupported --manager %q", opts.PackageManager)
}
ruleLine := ""
if !opts.NoRule {
addCmd := fmt.Sprintf("ip rule add to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Table)
if opts.Priority > 0 {
addCmd = fmt.Sprintf("ip rule add priority %d to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Priority, opts.Table)
}
ruleLine = fmt.Sprintf(`
if ! ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; then
%s
fi`, opts.Table, addCmd)
}
return fmt.Sprintf(`set -eu
PXMON_GATEWAY_HOST=%s
PXMON_GATEWAY_IP=%s
PXMON_PROXY_URL=%s
PXMON_MANAGER=%s
if [ -z "$PXMON_GATEWAY_IP" ]; then
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
fi
if [ -z "$PXMON_GATEWAY_IP" ]; then
echo "cannot resolve repo gateway: $PXMON_GATEWAY_HOST" >&2
exit 1
fi
%s
pxmon_repo_manager="$PXMON_MANAGER"
if [ "$pxmon_repo_manager" = "auto" ]; then
if command -v apt-get >/dev/null 2>&1; then pxmon_repo_manager=apt
elif command -v dnf >/dev/null 2>&1; then pxmon_repo_manager=dnf
elif command -v yum >/dev/null 2>&1; then pxmon_repo_manager=yum
else echo "no supported package manager found" >&2; exit 1
fi
fi
case "$pxmon_repo_manager" in
apt)
mkdir -p /etc/apt/apt.conf.d
cat > /etc/apt/apt.conf.d/99-pxmon-repo-tunnel <<EOF
Acquire::http::Proxy "$PXMON_PROXY_URL";
Acquire::https::Proxy "$PXMON_PROXY_URL";
EOF
;;
dnf|yum)
conf=/etc/dnf/dnf.conf
[ "$pxmon_repo_manager" = "yum" ] && conf=/etc/yum.conf
[ -f "$conf" ] || touch "$conf"
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
{
echo "# BEGIN PXMON REPO TUNNEL"
echo "proxy=$PXMON_PROXY_URL"
echo "# END PXMON REPO TUNNEL"
} >> "$conf"
;;
esac
echo "pxmon repo tunnel enabled: proxy=$PXMON_PROXY_URL gateway_ip=$PXMON_GATEWAY_IP manager=$pxmon_repo_manager"
`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), shellQuote(gw.proxyURL), shellQuote(manager), ruleLine), nil
}
func repoTunnelDisableScript(opts RepoTunnelOptions) (string, error) {
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err != nil && !opts.NoRule {
return "", err
}
ruleLine := ""
if !opts.NoRule {
if opts.Table <= 0 {
return "", errors.New("--table is required unless --no-rule is used")
}
ruleLine = fmt.Sprintf(`
PXMON_GATEWAY_HOST=%s
PXMON_GATEWAY_IP=%s
if [ -z "$PXMON_GATEWAY_IP" ]; then
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
fi
if [ -n "$PXMON_GATEWAY_IP" ]; then
while ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; do
ip rule del to "$PXMON_GATEWAY_IP/32" table %d 2>/dev/null || break
done
fi`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), opts.Table, opts.Table)
}
return fmt.Sprintf(`set -eu
rm -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
if [ -f "$conf" ]; then
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
fi
done
%s
echo "pxmon repo tunnel disabled"
`, ruleLine), nil
}
func repoTunnelStatusScript() string {
return `set -eu
echo "== ip rules =="
ip rule show | grep -E "lookup|table" || true
echo
echo "== apt proxy =="
[ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ] && cat /etc/apt/apt.conf.d/99-pxmon-repo-tunnel || echo "(none)"
echo
echo "== dnf/yum proxy =="
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
[ -f "$conf" ] || continue
echo "-- $conf"
awk '/# BEGIN PXMON REPO TUNNEL/,/# END PXMON REPO TUNNEL/ {print}' "$conf"
done`
}
func repoTunnelDetectScript() string {
return `set -eu
if [ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ]; then
proxy=$(sed -n 's/.*Proxy[[:space:]]*"\([^"]*\)".*/\1/p' /etc/apt/apt.conf.d/99-pxmon-repo-tunnel | head -1)
[ -n "$proxy" ] && printf 'proxy=%s\nsource=apt\n' "$proxy" && exit 0
fi
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
[ -f "$conf" ] || continue
proxy=$(awk '
/# BEGIN PXMON REPO TUNNEL/ {inside=1; next}
/# END PXMON REPO TUNNEL/ {inside=0; next}
inside && /^proxy[[:space:]]*=/ {
sub(/^[^=]*=/, "")
gsub(/^[[:space:]]+|[[:space:]]+$/, "")
print
exit
}
' "$conf")
[ -n "$proxy" ] && printf 'proxy=%s\nsource=%s\n' "$proxy" "$conf" && exit 0
done
exit 0`
}
type repoTunnelGateway struct {
host string
port int
proxyURL string
}
func parseRepoTunnelGateway(raw string) (repoTunnelGateway, error) {
v := strings.TrimSpace(raw)
if v == "" {
return repoTunnelGateway{}, errors.New("--gateway is required")
}
if strings.HasPrefix(v, "http://") {
v = strings.TrimPrefix(v, "http://")
}
if strings.HasPrefix(v, "https://") {
return repoTunnelGateway{}, errors.New("--gateway must be an http proxy endpoint, not https")
}
host, portRaw, err := net.SplitHostPort(v)
if err != nil {
if strings.Count(v, ":") > 1 {
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway %q; use host:port or [ipv6]:port", raw)
}
host = v
portRaw = "3128"
}
host = strings.Trim(host, "[]")
if strings.TrimSpace(host) == "" {
return repoTunnelGateway{}, errors.New("--gateway host is empty")
}
port, err := strconv.Atoi(portRaw)
if err != nil || port < 1 || port > 65535 {
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway port %q", portRaw)
}
return repoTunnelGateway{
host: host,
port: port,
proxyURL: "http://" + net.JoinHostPort(host, strconv.Itoa(port)),
}, nil
}
internal/cluster/runbook.go
+187
View File
@@ -0,0 +1,187 @@
package cluster
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
type RunbookStep struct {
Title string `json:"title"`
Command string `json:"command,omitempty"`
Note string `json:"note,omitempty"`
}
type Runbook struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Steps []RunbookStep `json:"steps"`
BuiltIn bool `json:"built_in,omitempty"`
CreatedAt time.Time `json:"created_at,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
func builtinRunbooks() []Runbook {
return []Runbook{
{
ID: "vm-health-check",
Name: "VM Health Check",
Description: "Quick validation of agent, VM inventory and VM alert policy.",
BuiltIn: true,
Steps: []RunbookStep{
{Title: "Check agent status", Command: "cluster agent status"},
{Title: "Check cluster drift", Command: "cluster drift"},
{Title: "Check VM states", Command: "cluster alert-vm check"},
{Title: "Inspect VM allocations", Command: "kvm top"},
},
},
{
ID: "traffic-billing-audit",
Name: "Traffic Billing Audit",
Description: "Collect interface P95 and graph evidence for billing period.",
BuiltIn: true,
Steps: []RunbookStep{
{Title: "List interfaces", Command: "cluster usage --range 1h"},
{Title: "Compute P95 for target iface", Command: "cluster p95 --iface <iface> --range 30d --graph"},
{Title: "Export report", Command: "cluster report export --format json --out ./report.json"},
},
},
}
}
func (s *Service) runbookPath() string {
return filepath.Join(s.DataDir(), "runbooks", "custom.json")
}
func (s *Service) loadCustomRunbooks() ([]Runbook, error) {
path := s.runbookPath()
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return []Runbook{}, nil
}
return nil, err
}
if len(strings.TrimSpace(string(raw))) == 0 {
return []Runbook{}, nil
}
var items []Runbook
if err := json.Unmarshal(raw, &items); err != nil {
return nil, err
}
out := make([]Runbook, 0, len(items))
for _, rb := range items {
rb.ID = strings.TrimSpace(rb.ID)
rb.Name = strings.TrimSpace(rb.Name)
if rb.ID == "" || rb.Name == "" {
continue
}
rb.BuiltIn = false
out = append(out, rb)
}
return out, nil
}
func (s *Service) saveCustomRunbooks(items []Runbook) error {
path := s.runbookPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
b, err := json.MarshalIndent(items, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
tmp := path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *Service) ListRunbooks() ([]Runbook, error) {
built := builtinRunbooks()
custom, err := s.loadCustomRunbooks()
if err != nil {
return nil, err
}
all := append(append([]Runbook{}, built...), custom...)
sort.Slice(all, func(i, j int) bool { return strings.ToLower(all[i].ID) < strings.ToLower(all[j].ID) })
return all, nil
}
func (s *Service) GetRunbook(selector string) (Runbook, bool) {
selector = strings.TrimSpace(selector)
items, err := s.ListRunbooks()
if err != nil {
return Runbook{}, false
}
for _, rb := range items {
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
return rb, true
}
}
return Runbook{}, false
}
func (s *Service) AddRunbook(rb Runbook) (Runbook, error) {
rb.ID = strings.TrimSpace(rb.ID)
rb.Name = strings.TrimSpace(rb.Name)
if rb.ID == "" || rb.Name == "" {
return Runbook{}, errors.New("runbook id and name are required")
}
if len(rb.Steps) == 0 {
return Runbook{}, errors.New("runbook steps are required")
}
if b, ok := s.GetRunbook(rb.ID); ok && b.BuiltIn {
return Runbook{}, errors.New("cannot overwrite built-in runbook")
}
custom, err := s.loadCustomRunbooks()
if err != nil {
return Runbook{}, err
}
for _, it := range custom {
if strings.EqualFold(it.ID, rb.ID) {
return Runbook{}, errors.New("runbook id already exists")
}
}
now := s.now().UTC()
rb.BuiltIn = false
rb.CreatedAt = now
rb.UpdatedAt = now
custom = append(custom, rb)
if err := s.saveCustomRunbooks(custom); err != nil {
return Runbook{}, err
}
_ = s.AppendChange("runbook.add", rb.ID, rb.Name)
return rb, nil
}
func (s *Service) RemoveRunbook(selector string) (Runbook, error) {
custom, err := s.loadCustomRunbooks()
if err != nil {
return Runbook{}, err
}
idx := -1
for i, rb := range custom {
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return Runbook{}, errors.New("custom runbook not found")
}
removed := custom[idx]
custom = append(custom[:idx], custom[idx+1:]...)
if err := s.saveCustomRunbooks(custom); err != nil {
return Runbook{}, err
}
_ = s.AppendChange("runbook.remove", removed.ID, removed.Name)
return removed, nil
}
internal/cluster/scheduler.go
+301
View File
@@ -0,0 +1,301 @@
package cluster
import (
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
type ScheduledTask struct {
ID string `json:"id"`
Name string `json:"name"`
Cluster string `json:"cluster,omitempty"`
Command string `json:"command"`
Mode string `json:"mode,omitempty"` // shell|observer
Every string `json:"every"`
Backoff string `json:"backoff,omitempty"`
JitterSec int `json:"jitter_sec,omitempty"`
RetryMax int `json:"retry_max,omitempty"`
RetryCur int `json:"retry_cur,omitempty"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
LastRunAt time.Time `json:"last_run_at,omitempty"`
NextRunAt time.Time `json:"next_run_at,omitempty"`
}
type SchedulerRunResult struct {
Task ScheduledTask `json:"task"`
Ran bool `json:"ran"`
Error string `json:"error,omitempty"`
Output string `json:"output,omitempty"`
ExitCode int `json:"exit_code,omitempty"`
}
func (s *Service) schedulerPath() string {
return filepath.Join(s.DataDir(), "scheduler", "tasks.json")
}
func (s *Service) loadTasks() ([]ScheduledTask, error) {
path := s.schedulerPath()
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return []ScheduledTask{}, nil
}
return nil, err
}
var items []ScheduledTask
if len(strings.TrimSpace(string(raw))) == 0 {
return []ScheduledTask{}, nil
}
if err := json.Unmarshal(raw, &items); err != nil {
return nil, err
}
for i := range items {
items[i] = normalizeScheduledTask(items[i])
}
sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
return items, nil
}
func (s *Service) saveTasks(items []ScheduledTask) error {
path := s.schedulerPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
b, err := json.MarshalIndent(items, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
tmp := path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func normalizeScheduledTask(t ScheduledTask) ScheduledTask {
t.ID = strings.TrimSpace(t.ID)
t.Name = strings.TrimSpace(t.Name)
t.Command = strings.TrimSpace(t.Command)
t.Cluster = strings.TrimSpace(t.Cluster)
t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
t.Every = strings.TrimSpace(t.Every)
if t.ID == "" {
t.ID = newClusterID()
}
if t.Name == "" {
t.Name = t.ID
}
if t.Every == "" {
t.Every = "5m"
}
if strings.TrimSpace(t.Backoff) == "" {
t.Backoff = "30s"
}
if t.JitterSec < 0 {
t.JitterSec = 0
}
if t.RetryMax <= 0 {
t.RetryMax = 3
}
if t.RetryCur < 0 {
t.RetryCur = 0
}
if t.Mode == "" {
t.Mode = "shell"
}
if t.Mode != "shell" && t.Mode != "observer" {
t.Mode = "shell"
}
if t.CreatedAt.IsZero() {
t.CreatedAt = time.Now().UTC()
}
if t.UpdatedAt.IsZero() {
t.UpdatedAt = t.CreatedAt
}
return t
}
func parseTaskEvery(v string) (time.Duration, error) {
d, err := time.ParseDuration(strings.TrimSpace(v))
if err != nil {
return 0, fmt.Errorf("invalid --every duration: %w", err)
}
if d < time.Minute {
return 0, errors.New("--every must be >= 1m")
}
return d, nil
}
func (s *Service) SchedulerList() ([]ScheduledTask, error) {
return s.loadTasks()
}
func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) {
name = strings.TrimSpace(name)
command = strings.TrimSpace(command)
every = strings.TrimSpace(every)
mode = strings.ToLower(strings.TrimSpace(mode))
if name == "" {
return ScheduledTask{}, errors.New("task name is required")
}
if command == "" {
return ScheduledTask{}, errors.New("task command is required")
}
if mode == "" {
mode = "shell"
}
if mode != "shell" && mode != "observer" {
return ScheduledTask{}, errors.New("task mode must be shell|observer")
}
d, err := parseTaskEvery(every)
if err != nil {
return ScheduledTask{}, err
}
if strings.TrimSpace(backoff) == "" {
backoff = "30s"
}
if _, err := time.ParseDuration(backoff); err != nil {
return ScheduledTask{}, errors.New("invalid backoff duration")
}
if jitterSec < 0 {
jitterSec = 0
}
if retryMax <= 0 {
retryMax = 3
}
items, err := s.loadTasks()
if err != nil {
return ScheduledTask{}, err
}
for _, t := range items {
if strings.EqualFold(t.Name, name) {
return ScheduledTask{}, fmt.Errorf("task %q already exists", name)
}
}
now := s.now().UTC()
t := ScheduledTask{
ID: newClusterID(),
Name: name,
Cluster: strings.TrimSpace(clusterSel),
Command: command,
Mode: mode,
Every: every,
Backoff: backoff,
JitterSec: jitterSec,
RetryMax: retryMax,
Enabled: enabled,
CreatedAt: now,
UpdatedAt: now,
}
if enabled {
t.NextRunAt = now.Add(d)
}
items = append(items, t)
if err := s.saveTasks(items); err != nil {
return ScheduledTask{}, err
}
_ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every))
return t, nil
}
func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) {
items, err := s.loadTasks()
if err != nil {
return ScheduledTask{}, err
}
selector = strings.TrimSpace(selector)
if selector == "" {
return ScheduledTask{}, errors.New("task name or id is required")
}
idx := -1
for i, t := range items {
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return ScheduledTask{}, errors.New("task not found")
}
removed := items[idx]
items = append(items[:idx], items[idx+1:]...)
if err := s.saveTasks(items); err != nil {
return ScheduledTask{}, err
}
_ = s.AppendChange("scheduler.remove", removed.Name, removed.Command)
return removed, nil
}
func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error {
items, err := s.loadTasks()
if err != nil {
return err
}
for i := range items {
if items[i].ID != taskID {
continue
}
items[i].LastRunAt = ranAt.UTC()
if success {
d, err := parseTaskEvery(items[i].Every)
if err != nil {
return err
}
items[i].RetryCur = 0
items[i].NextRunAt = items[i].LastRunAt.Add(d)
} else {
items[i].RetryCur++
if items[i].RetryCur > items[i].RetryMax {
// cap retries and move to the next normal run window
items[i].RetryCur = 0
d, err := parseTaskEvery(items[i].Every)
if err != nil {
return err
}
items[i].NextRunAt = items[i].LastRunAt.Add(d)
} else {
back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff))
if err != nil || back <= 0 {
back = 30 * time.Second
}
delay := back * time.Duration(1<<(items[i].RetryCur-1))
if items[i].JitterSec > 0 {
delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second
}
items[i].NextRunAt = items[i].LastRunAt.Add(delay)
}
}
items[i].UpdatedAt = ranAt.UTC()
break
}
return s.saveTasks(items)
}
func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) {
items, err := s.loadTasks()
if err != nil {
return nil, err
}
n := now.UTC()
due := make([]ScheduledTask, 0)
for _, t := range items {
if !t.Enabled {
continue
}
if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) {
due = append(due, t)
}
}
sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) })
return due, nil
}
internal/cluster/service.go
+2602
View File
File diff suppressed because it is too large Load Diff
internal/cluster/service_test.go
+554
View File
@@ -0,0 +1,554 @@
package cluster
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
func newTestService(t *testing.T) *Service {
t.Helper()
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
return NewService(store)
}
func TestConnectListUseDisconnect(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
c1, _, err := svc.Connect(ctx, ConnectOptions{
Name: "eu-1",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass1",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect c1 error: %v", err)
}
c2, _, err := svc.Connect(ctx, ConnectOptions{
Name: "us-1",
Host: "10.0.0.11",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect c2 error: %v", err)
}
clusters, activeID, err := svc.List()
if err != nil {
t.Fatalf("list error: %v", err)
}
if len(clusters) != 2 {
t.Fatalf("expected 2 clusters, got %d", len(clusters))
}
if activeID != c2.ID {
t.Fatalf("expected active %s, got %s", c2.ID, activeID)
}
_, err = svc.Use(c1.Name)
if err != nil {
t.Fatalf("use error: %v", err)
}
current, err := svc.Current()
if err != nil {
t.Fatalf("current error: %v", err)
}
if current.ID != c1.ID {
t.Fatalf("expected current %s, got %s", c1.ID, current.ID)
}
_, err = svc.Disconnect(c1.Name)
if err != nil {
t.Fatalf("disconnect error: %v", err)
}
current, err = svc.Current()
if err != nil {
t.Fatalf("current after disconnect error: %v", err)
}
if current.ID != c2.ID {
t.Fatalf("expected fallback current %s, got %s", c2.ID, current.ID)
}
}
func TestConnectDuplicateNameRequiresForce(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass1",
SkipCheck: true,
})
if err != nil {
t.Fatalf("initial connect error: %v", err)
}
_, _, err = svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.20",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
})
if err == nil {
t.Fatal("expected duplicate name error")
}
c, _, err := svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.20",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
Force: true,
})
if err != nil {
t.Fatalf("force connect error: %v", err)
}
if c.Host != "10.0.0.20" {
t.Fatalf("expected overwritten host, got %s", c.Host)
}
}
func TestExportImportRestoresKeyFilesAndNewFields(t *testing.T) {
t.Parallel()
srcDir := t.TempDir()
keyPath := filepath.Join(srcDir, "id_ed25519")
passPath := filepath.Join(srcDir, "pass.pxmonpassphrase")
sftpKeyPath := filepath.Join(srcDir, "sftp_key")
if err := os.WriteFile(keyPath, []byte("PRIVATE KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(passPath, []byte("secret-pass\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(sftpKeyPath, []byte("SFTP PRIVATE KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
src := newTestService(t)
reg := newRegistry()
reg.ActiveClusterID = "clu_1"
reg.Backups.Targets = []BackupTarget{{
ID: "bt_1",
Name: "sftp",
Type: "sftp",
Enabled: true,
SFTPKeyPath: sftpKeyPath,
}}
reg.Clusters = []Cluster{{
ID: "clu_1",
Name: "node",
Host: "192.0.2.10",
Port: 22,
User: "root",
Transport: TransportIPFabric,
AuthMethod: AuthMethodKey,
KeyPath: keyPath,
KeyPassphraseFile: passPath,
RepoTunnel: RepoTunnelState{
Enabled: true,
Proxy: "http://203.0.113.10:3128",
Source: "dnf",
},
Alerts: defaultAlertPolicy(),
CreatedAt: time.Now().UTC(),
UpdatedAt: time.Now().UTC(),
}}
if err := src.store.Save(reg); err != nil {
t.Fatalf("save source registry: %v", err)
}
bundlePath := filepath.Join(t.TempDir(), "pxmon-export.enc")
if err := src.Export(bundlePath, "strong-test-passphrase"); err != nil {
t.Fatalf("export error: %v", err)
}
raw, err := os.ReadFile(bundlePath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "PRIVATE KEY") || strings.Contains(string(raw), "secret-pass") {
t.Fatal("export bundle leaked key material in plaintext")
}
dst := newTestService(t)
if _, err := dst.Import(bundlePath, "strong-test-passphrase", ImportModeReplace); err != nil {
t.Fatalf("import error: %v", err)
}
gotReg, err := dst.store.Load()
if err != nil {
t.Fatalf("load imported registry: %v", err)
}
if len(gotReg.Clusters) != 1 {
t.Fatalf("expected 1 cluster, got %d", len(gotReg.Clusters))
}
got := gotReg.Clusters[0]
if !got.RepoTunnel.Enabled || got.RepoTunnel.Proxy != "http://203.0.113.10:3128" {
t.Fatalf("repo tunnel was not preserved: %+v", got.RepoTunnel)
}
if got.KeyPath == keyPath || got.KeyPassphraseFile == passPath {
t.Fatalf("expected key paths to be restored under destination config dir, got key=%q pass=%q", got.KeyPath, got.KeyPassphraseFile)
}
keyData, err := os.ReadFile(got.KeyPath)
if err != nil {
t.Fatalf("read restored key: %v", err)
}
if string(keyData) != "PRIVATE KEY\n" {
t.Fatalf("unexpected restored key content: %q", string(keyData))
}
passData, err := os.ReadFile(got.KeyPassphraseFile)
if err != nil {
t.Fatalf("read restored passphrase file: %v", err)
}
if string(passData) != "secret-pass\n" {
t.Fatalf("unexpected restored passphrase content: %q", string(passData))
}
if len(gotReg.Backups.Targets) != 1 || gotReg.Backups.Targets[0].SFTPKeyPath == sftpKeyPath {
t.Fatalf("expected restored SFTP key path, got %+v", gotReg.Backups.Targets)
}
mergeDst := newTestService(t)
if _, err := mergeDst.Import(bundlePath, "strong-test-passphrase", ImportModeMerge); err != nil {
t.Fatalf("merge import error: %v", err)
}
mergeReg, err := mergeDst.store.Load()
if err != nil {
t.Fatalf("load merge registry: %v", err)
}
if len(mergeReg.Backups.Targets) != 1 || strings.TrimSpace(mergeReg.Backups.Targets[0].SFTPKeyPath) == "" {
t.Fatalf("expected backup target to be merged, got %+v", mergeReg.Backups)
}
}
func TestConnectRequiresAuthData(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "bad",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
SkipCheck: true,
})
if err == nil {
t.Fatal("expected error for empty password auth")
}
}
func TestAlertPolicySetAndGet(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "node-1",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect error: %v", err)
}
updated, err := svc.SetAlertPolicy("node-1", AlertPolicy{
CPUWarnPercent: 70,
RAMWarnPercent: 75,
SwapWarnPercent: 60,
DiskWarnPercent: 80,
NetWarnMbps: 120,
NetSustainEnabled: true,
NetSustainIface: "eth0",
NetSustainInclude: []string{"net0"},
NetSustainExclude: []string{"backup"},
NetSustainMbps: 500,
NetSustainMinutes: 60,
NetSustainCooldownMins: 15,
})
if err != nil {
t.Fatalf("set alert policy error: %v", err)
}
if updated.Alerts.NetWarnMbps != 120 {
t.Fatalf("unexpected net threshold: %.2f", updated.Alerts.NetWarnMbps)
}
got, err := svc.GetAlertPolicy("node-1")
if err != nil {
t.Fatalf("get alert policy error: %v", err)
}
if got.RAMWarnPercent != 75 || got.DiskWarnPercent != 80 {
t.Fatalf("unexpected thresholds: %+v", got)
}
if !got.NetSustainEnabled || got.NetSustainIface != "eth0" || got.NetSustainMbps != 500 {
t.Fatalf("unexpected sustained net policy: %+v", got)
}
if len(got.NetSustainInclude) != 1 || got.NetSustainInclude[0] != "net0" {
t.Fatalf("unexpected sustained net include filter: %+v", got.NetSustainInclude)
}
if len(got.NetSustainExclude) != 1 || got.NetSustainExclude[0] != "backup" {
t.Fatalf("unexpected sustained net exclude filter: %+v", got.NetSustainExclude)
}
}
func TestIsPluginToolSupported(t *testing.T) {
t.Parallel()
info := SoftwareInfo{
Bird: true,
FRR: true,
KVM: true,
LXC: true,
LXD: false,
}
tests := []struct {
tool string
ok bool
}{
{tool: "bird", ok: true},
{tool: "frr", ok: true},
{tool: "kvm", ok: true},
{tool: "lxc", ok: true},
{tool: "lxd", ok: true}, // lxd aliases to lxc command templates
{tool: "unknown", ok: false},
}
for _, tt := range tests {
tt := tt
t.Run(tt.tool, func(t *testing.T) {
t.Parallel()
if got := isPluginToolSupported(info, tt.tool); got != tt.ok {
t.Fatalf("tool=%s expected %v got %v", tt.tool, tt.ok, got)
}
})
}
}
func TestLXDTopUsesTemplateNotRawLxcTop(t *testing.T) {
t.Parallel()
script, err := pluginScript("lxd", "top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
if strings.Contains(script, "lxc top") {
t.Fatalf("expected custom template script, got raw lxc top: %q", script)
}
if !strings.Contains(script, "lxc info") {
t.Fatalf("expected lxc info usage in top template")
}
}
func TestRunPluginActionReportsMissingSupport(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
cluster, _, err := svc.Connect(ctx, ConnectOptions{
Name: "eu-1",
Host: "127.0.0.1",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect error: %v", err)
}
reg, err := svc.store.Load()
if err != nil {
t.Fatalf("load registry error: %v", err)
}
for i := range reg.Clusters {
if reg.Clusters[i].ID == cluster.ID {
reg.Clusters[i].Software = SoftwareInfo{
DetectedAt: time.Now().UTC(),
}
break
}
}
if err := svc.store.Save(reg); err != nil {
t.Fatalf("save registry error: %v", err)
}
_, err = svc.RunPluginAction(ctx, cluster.Name, "lxd", "top", nil)
if err == nil {
t.Fatalf("expected unsupported software error")
}
if !strings.Contains(err.Error(), "support for lxd was not detected") {
t.Fatalf("unexpected error message: %v", err)
}
}
func TestKVMTopScriptIncludesReadableMetrics(t *testing.T) {
t.Parallel()
script, err := pluginScript("kvm", "top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
for _, want := range []string{
"VCPU",
"RAM_MAX",
"DISK_CAP",
"DISK_ALLOC",
} {
if !strings.Contains(script, want) {
t.Fatalf("expected %q in kvm top script", want)
}
}
}
func TestKVMNetTopScriptIncludesRateAndP95(t *testing.T) {
t.Parallel()
script, err := pluginScript("kvm", "net-top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
for _, want := range []string{
"NET_Mbps",
"RX_Mbps",
"TX_Mbps",
"P95_Mbps",
"RX_TOTAL",
"TX_TOTAL",
} {
if !strings.Contains(script, want) {
t.Fatalf("expected %q in kvm net-top script", want)
}
}
}
func TestKVMScriptsAreShellParseable(t *testing.T) {
t.Parallel()
cases := []struct {
tool string
action string
}{
{tool: "kvm", action: "top"},
{tool: "kvm", action: "net-top"},
}
for _, tc := range cases {
tc := tc
t.Run(tc.tool+"-"+tc.action, func(t *testing.T) {
t.Parallel()
script, err := pluginScript(tc.tool, tc.action, nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
cmd := exec.Command("sh", "-n", "-c", script)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("shell parse failed: %v\n%s\nSCRIPT:\n%s", err, string(out), script)
}
})
}
}
func TestTelegramConfigSetGetDisable(t *testing.T) {
t.Parallel()
svc := newTestService(t)
cfg, err := svc.SetTelegram(Telegram{
Enabled: true,
Token: "123:ABC",
AllowedUserIDs: []int64{2002, 1001, 2002},
})
if err != nil {
t.Fatalf("set telegram config: %v", err)
}
if !cfg.Enabled {
t.Fatal("expected enabled telegram config")
}
if len(cfg.AllowedUserIDs) != 2 {
t.Fatalf("expected deduped ids, got %+v", cfg.AllowedUserIDs)
}
got, err := svc.GetTelegram()
if err != nil {
t.Fatalf("get telegram config: %v", err)
}
if got.Token != "123:ABC" {
t.Fatalf("unexpected token: %q", got.Token)
}
if len(got.AllowedUserIDs) != 2 || got.AllowedUserIDs[0] != 1001 || got.AllowedUserIDs[1] != 2002 {
t.Fatalf("unexpected allowed ids: %+v", got.AllowedUserIDs)
}
disabled, err := svc.DisableTelegram()
if err != nil {
t.Fatalf("disable telegram config: %v", err)
}
if disabled.Enabled {
t.Fatal("expected disabled telegram config")
}
}
func TestTelegramConfigValidation(t *testing.T) {
t.Parallel()
svc := newTestService(t)
_, err := svc.SetTelegram(Telegram{
Enabled: true,
AllowedUserIDs: []int64{123},
})
if err == nil {
t.Fatal("expected token validation error")
}
_, err = svc.SetTelegram(Telegram{
Enabled: true,
Token: "123:ABC",
})
if err == nil {
t.Fatal("expected allowed ids validation error")
}
}
internal/cluster/slo_capacity.go
+159
View File
@@ -0,0 +1,159 @@
package cluster
import (
"math"
"sort"
"strings"
"time"
"pxmon/internal/history"
)
type AvailabilityVM struct {
Name string `json:"name"`
Availability float64 `json:"availability_pct"`
Samples int `json:"samples"`
Running int `json:"running_samples"`
}
type AvailabilityReport struct {
Cluster string `json:"cluster"`
Range string `json:"range"`
Samples int `json:"samples"`
UpSamples int `json:"up_samples"`
Availability float64 `json:"availability_pct"`
VMs []AvailabilityVM `json:"vms,omitempty"`
}
func (s *Service) AvailabilityReport(selector string, since time.Time, vmFilter string) (AvailabilityReport, error) {
c, err := s.Get(selector)
if err != nil {
return AvailabilityReport{}, err
}
store := history.NewAvailabilityStore(s.DataDir())
snaps, err := store.Load(c.ID, since)
if err != nil {
return AvailabilityReport{}, err
}
rep := AvailabilityReport{Cluster: c.Name, Samples: len(snaps)}
if len(snaps) == 0 {
return rep, nil
}
vmFilter = strings.TrimSpace(vmFilter)
totalUp := 0
type acc struct{ samples, running int }
vmap := map[string]*acc{}
for _, snap := range snaps {
if snap.ClusterUp {
totalUp++
}
for vm, st := range snap.VMStates {
if vmFilter != "" && !strings.EqualFold(vmFilter, vm) {
continue
}
a := vmap[vm]
if a == nil {
a = &acc{}
vmap[vm] = a
}
a.samples++
if strings.EqualFold(strings.TrimSpace(st), "running") {
a.running++
}
}
}
rep.UpSamples = totalUp
rep.Availability = 100 * float64(totalUp) / float64(len(snaps))
for vm, a := range vmap {
if a.samples == 0 {
continue
}
rep.VMs = append(rep.VMs, AvailabilityVM{
Name: vm,
Samples: a.samples,
Running: a.running,
Availability: 100 * float64(a.running) / float64(a.samples),
})
}
sort.Slice(rep.VMs, func(i, j int) bool { return rep.VMs[i].Name < rep.VMs[j].Name })
return rep, nil
}
type CapacityForecastItem struct {
Mount string `json:"mount"`
UsedPct float64 `json:"used_pct"`
SlopeBytesSec float64 `json:"slope_bytes_per_sec"`
DaysTo90 float64 `json:"days_to_90_pct"`
DaysTo95 float64 `json:"days_to_95_pct"`
}
type CapacityForecastReport struct {
Cluster string `json:"cluster"`
Samples int `json:"samples"`
Items []CapacityForecastItem `json:"items,omitempty"`
}
func (s *Service) CapacityForecast(selector string, since time.Time) (CapacityForecastReport, error) {
c, err := s.Get(selector)
if err != nil {
return CapacityForecastReport{}, err
}
store := history.NewCapacityStore(s.DataDir())
snaps, err := store.Load(c.ID, since)
if err != nil {
return CapacityForecastReport{}, err
}
rep := CapacityForecastReport{Cluster: c.Name, Samples: len(snaps)}
if len(snaps) < 2 {
return rep, nil
}
type point struct {
ts time.Time
used float64
tot float64
}
byMount := map[string][]point{}
for _, snap := range snaps {
for _, d := range snap.Disks {
if d.TotalBytes == 0 {
continue
}
byMount[d.Mount] = append(byMount[d.Mount], point{ts: snap.Timestamp, used: float64(d.UsedBytes), tot: float64(d.TotalBytes)})
}
}
for mnt, pts := range byMount {
if len(pts) < 2 {
continue
}
sort.Slice(pts, func(i, j int) bool { return pts[i].ts.Before(pts[j].ts) })
first := pts[0]
last := pts[len(pts)-1]
dt := last.ts.Sub(first.ts).Seconds()
if dt <= 0 {
continue
}
slope := (last.used - first.used) / dt
usedPct := 100 * last.used / last.tot
d90 := daysToTarget(last.used, last.tot*0.90, slope)
d95 := daysToTarget(last.used, last.tot*0.95, slope)
rep.Items = append(rep.Items, CapacityForecastItem{
Mount: mnt,
UsedPct: usedPct,
SlopeBytesSec: slope,
DaysTo90: d90,
DaysTo95: d95,
})
}
sort.Slice(rep.Items, func(i, j int) bool { return rep.Items[i].UsedPct > rep.Items[j].UsedPct })
return rep, nil
}
func daysToTarget(current, target, slope float64) float64 {
if target <= current {
return 0
}
if slope <= 0 {
return math.Inf(1)
}
return (target - current) / slope / 86400
}
internal/cluster/store.go
+425
View File
@@ -0,0 +1,425 @@
package cluster
import (
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
envConfigPath = "PXMON_CONFIG"
envMasterKeyPath = "PXMON_MASTER_KEY"
filePrefix = "OBSCTL1:"
masterKeyBytes = 32
)
// Store persists encrypted cluster registry on disk.
type Store struct {
path string
keyPath string
}
func NewStore(path string) (*Store, error) {
if path == "" {
var err error
path, err = DefaultConfigPath()
if err != nil {
return nil, err
}
}
keyPath, err := defaultMasterKeyPath(path)
if err != nil {
return nil, err
}
return &Store{path: path, keyPath: keyPath}, nil
}
func DefaultConfigPath() (string, error) {
if p := os.Getenv(envConfigPath); p != "" {
return p, nil
}
dir, err := os.UserConfigDir()
if err != nil {
return "", fmt.Errorf("resolve user config dir: %w", err)
}
return filepath.Join(dir, "pxmon", "clusters.enc"), nil
}
func defaultMasterKeyPath(configPath string) (string, error) {
if p := os.Getenv(envMasterKeyPath); p != "" {
return p, nil
}
if configPath == "" {
return "", errors.New("empty config path")
}
return filepath.Join(filepath.Dir(configPath), "master.key"), nil
}
func (s *Store) Path() string {
return s.path
}
func (s *Store) KeyPath() string {
return s.keyPath
}
func (s *Store) LockerSessionPath() string {
return filepath.Join(filepath.Dir(s.path), "locker.session")
}
func (s *Store) LockerAuditPath() string {
return filepath.Join(filepath.Dir(s.path), "locker.audit.log")
}
func (s *Store) Load() (Registry, error) {
f, err := os.Open(s.path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return newRegistry(), nil
}
return Registry{}, fmt.Errorf("open registry file: %w", err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return Registry{}, fmt.Errorf("read registry file: %w", err)
}
if len(strings.TrimSpace(string(data))) == 0 {
return newRegistry(), nil
}
payload, err := s.decodePayload(data)
if err != nil {
return Registry{}, err
}
var reg Registry
if err := json.Unmarshal(payload, &reg); err != nil {
return Registry{}, fmt.Errorf("decode registry JSON: %w", err)
}
if reg.Version == 0 {
reg.Version = currentVersion
}
if reg.Clusters == nil {
reg.Clusters = []Cluster{}
}
for i := range reg.Clusters {
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
}
reg.Telegram = normalizeTelegram(reg.Telegram)
reg.Locker = normalizeLocker(reg.Locker)
reg.Backups = normalizeBackupConfig(reg.Backups)
return reg, nil
}
func (s *Store) decodePayload(data []byte) ([]byte, error) {
text := strings.TrimSpace(string(data))
if strings.HasPrefix(text, "{") {
// Backward compatibility with legacy unencrypted format.
return []byte(text), nil
}
if !strings.HasPrefix(text, filePrefix) {
return nil, errors.New("unsupported registry format")
}
blob := strings.TrimPrefix(text, filePrefix)
raw, err := base64.StdEncoding.DecodeString(blob)
if err != nil {
return nil, fmt.Errorf("decode encrypted payload: %w", err)
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return nil, err
}
payload, err := decrypt(raw, key)
if err != nil {
return nil, fmt.Errorf("decrypt registry: %w", err)
}
return payload, nil
}
func (s *Store) Save(reg Registry) error {
reg.Version = currentVersion
if reg.Clusters == nil {
reg.Clusters = []Cluster{}
}
for i := range reg.Clusters {
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
}
reg.Telegram = normalizeTelegram(reg.Telegram)
reg.Locker = normalizeLocker(reg.Locker)
reg.Backups = normalizeBackupConfig(reg.Backups)
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
return fmt.Errorf("create config dir: %w", err)
}
payload, err := json.MarshalIndent(reg, "", " ")
if err != nil {
return fmt.Errorf("encode registry JSON: %w", err)
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return err
}
encrypted, err := encrypt(payload, key)
if err != nil {
return fmt.Errorf("encrypt registry: %w", err)
}
content := filePrefix + base64.StdEncoding.EncodeToString(encrypted) + "\n"
tmp := s.path + ".tmp"
if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
return fmt.Errorf("write temp registry file: %w", err)
}
if err := os.Rename(tmp, s.path); err != nil {
return fmt.Errorf("replace registry file: %w", err)
}
return nil
}
func (s *Store) loadOrCreateMasterKey() ([]byte, error) {
if err := os.MkdirAll(filepath.Dir(s.keyPath), 0o700); err != nil {
return nil, fmt.Errorf("create key dir: %w", err)
}
key, err := os.ReadFile(s.keyPath)
if err == nil {
if len(key) != masterKeyBytes {
return nil, fmt.Errorf("invalid master key length: got %d", len(key))
}
return key, nil
}
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read master key: %w", err)
}
key = make([]byte, masterKeyBytes)
if _, err := rand.Read(key); err != nil {
return nil, fmt.Errorf("generate master key: %w", err)
}
tmp := s.keyPath + ".tmp"
if err := os.WriteFile(tmp, key, 0o600); err != nil {
return nil, fmt.Errorf("write temp master key: %w", err)
}
if err := os.Rename(tmp, s.keyPath); err != nil {
return nil, fmt.Errorf("replace master key: %w", err)
}
return key, nil
}
func encrypt(payload, key []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, err
}
sealed := gcm.Seal(nil, nonce, payload, nil)
out := make([]byte, 0, len(nonce)+len(sealed))
out = append(out, nonce...)
out = append(out, sealed...)
return out, nil
}
func decrypt(raw, key []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(raw) <= nonceSize {
return nil, errors.New("ciphertext too short")
}
nonce := raw[:nonceSize]
ciphertext := raw[nonceSize:]
payload, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, err
}
return payload, nil
}
type lockerSessionState struct {
ExpiresAtUnix int64 `json:"expires_at_unix"`
SigHex string `json:"sig_hex"`
}
func (s *Store) SaveLockerSession(passwordHash string, ttl time.Duration) error {
passwordHash = strings.TrimSpace(passwordHash)
if passwordHash == "" {
return errors.New("empty locker password hash")
}
if ttl <= 0 {
ttl = 6 * time.Hour
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return err
}
expires := time.Now().UTC().Add(ttl).Unix()
payload := strconv.FormatInt(expires, 10) + "|" + passwordHash
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(payload))
sig := hex.EncodeToString(mac.Sum(nil))
state := lockerSessionState{
ExpiresAtUnix: expires,
SigHex: sig,
}
data, err := json.Marshal(state)
if err != nil {
return err
}
data = append(data, '\n')
path := s.LockerSessionPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *Store) ValidateLockerSession(passwordHash string) (bool, time.Time, error) {
passwordHash = strings.TrimSpace(passwordHash)
if passwordHash == "" {
return false, time.Time{}, nil
}
path := s.LockerSessionPath()
raw, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, time.Time{}, nil
}
return false, time.Time{}, err
}
var state lockerSessionState
if err := json.Unmarshal(raw, &state); err != nil {
return false, time.Time{}, nil
}
if state.ExpiresAtUnix <= 0 || strings.TrimSpace(state.SigHex) == "" {
return false, time.Time{}, nil
}
expiresAt := time.Unix(state.ExpiresAtUnix, 0).UTC()
if time.Now().UTC().After(expiresAt) {
return false, expiresAt, nil
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return false, time.Time{}, err
}
payload := strconv.FormatInt(state.ExpiresAtUnix, 10) + "|" + passwordHash
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(payload))
expected := mac.Sum(nil)
got, err := hex.DecodeString(strings.TrimSpace(state.SigHex))
if err != nil {
return false, expiresAt, nil
}
if !hmac.Equal(expected, got) {
return false, expiresAt, nil
}
return true, expiresAt, nil
}
func (s *Store) ClearLockerSession() error {
path := s.LockerSessionPath()
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
func (s *Store) AppendLockerAudit(event, detail string) error {
path := s.LockerAuditPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
return err
}
defer f.Close()
ts := time.Now().UTC().Format(time.RFC3339)
event = strings.TrimSpace(event)
detail = strings.TrimSpace(detail)
if event == "" {
event = "event"
}
if detail == "" {
detail = "-"
}
_, err = fmt.Fprintf(f, "%s event=%s detail=%s\n", ts, event, strconv.Quote(detail))
return err
}
internal/cluster/store_test.go
+110
View File
@@ -0,0 +1,110 @@
package cluster
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestStoreLoadMissing(t *testing.T) {
t.Parallel()
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
reg, err := store.Load()
if err != nil {
t.Fatalf("Load error: %v", err)
}
if reg.Version != currentVersion {
t.Fatalf("expected version %d, got %d", currentVersion, reg.Version)
}
if len(reg.Clusters) != 0 {
t.Fatalf("expected empty clusters, got %d", len(reg.Clusters))
}
}
func TestStoreSaveLoadRoundtrip(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "clusters.enc")
store, err := NewStore(path)
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
seed := newRegistry()
seed.ActiveClusterID = "clu_1"
seed.Clusters = []Cluster{{
ID: "clu_1",
Name: "prod",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "secret123",
}}
if err := store.Save(seed); err != nil {
t.Fatalf("Save error: %v", err)
}
got, err := store.Load()
if err != nil {
t.Fatalf("Load error: %v", err)
}
if got.ActiveClusterID != "clu_1" {
t.Fatalf("active cluster mismatch: %s", got.ActiveClusterID)
}
if len(got.Clusters) != 1 || got.Clusters[0].Name != "prod" {
t.Fatalf("unexpected clusters: %+v", got.Clusters)
}
if got.Clusters[0].Password != "secret123" {
t.Fatalf("password mismatch after decrypt: %s", got.Clusters[0].Password)
}
}
func TestStorePersistsEncryptedPayload(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "clusters.enc")
store, err := NewStore(path)
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
reg := newRegistry()
reg.Clusters = []Cluster{{
ID: "clu_1",
Name: "sensitive-prod",
Host: "192.168.1.1",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "very-secret",
}}
if err := store.Save(reg); err != nil {
t.Fatalf("Save error: %v", err)
}
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile error: %v", err)
}
if !strings.HasPrefix(string(raw), filePrefix) {
t.Fatalf("expected encrypted file prefix %q", filePrefix)
}
if strings.Contains(string(raw), "sensitive-prod") || strings.Contains(string(raw), "very-secret") {
t.Fatal("plaintext secrets leaked into encrypted file")
}
if _, err := os.Stat(store.KeyPath()); err != nil {
t.Fatalf("master key not created: %v", err)
}
}
internal/cluster/tags.go
+144
View File
@@ -0,0 +1,144 @@
package cluster
import (
"errors"
"sort"
"strings"
)
func (s *Service) AddClusterTags(selector string, tags []string) (Cluster, error) {
return s.updateClusterTags(selector, tags, true)
}
func (s *Service) RemoveClusterTags(selector string, tags []string) (Cluster, error) {
return s.updateClusterTags(selector, tags, false)
}
func (s *Service) updateClusterTags(selector string, tags []string, add bool) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
changes := make(map[string]struct{}, len(tags))
for _, t := range normalizeTagList(tags) {
changes[t] = struct{}{}
}
if len(changes) == 0 {
return c, errors.New("at least one non-empty tag is required")
}
current := make(map[string]struct{}, len(c.Tags))
for _, t := range normalizeTagList(c.Tags) {
current[t] = struct{}{}
}
if add {
for t := range changes {
current[t] = struct{}{}
}
} else {
for t := range changes {
delete(current, t)
}
}
out := make([]string, 0, len(current))
for t := range current {
out = append(out, t)
}
sort.Strings(out)
c.Tags = out
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("cluster.tags", c.Name, strings.Join(out, ","))
return c, nil
}
func (s *Service) ListClusterTags(selector string) ([]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
return normalizeTagList(c.Tags), nil
}
func (s *Service) AddKVMTag(selector, vm string, tags []string) (Cluster, error) {
return s.updateKVMTags(selector, vm, tags, true)
}
func (s *Service) RemoveKVMTag(selector, vm string, tags []string) (Cluster, error) {
return s.updateKVMTags(selector, vm, tags, false)
}
func (s *Service) updateKVMTags(selector, vm string, tags []string, add bool) (Cluster, error) {
vm = strings.TrimSpace(vm)
if vm == "" {
return Cluster{}, errors.New("vm name is required")
}
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
if c.KVMTags == nil {
c.KVMTags = map[string][]string{}
}
current := make(map[string]struct{}, len(c.KVMTags[vm]))
for _, t := range normalizeTagList(c.KVMTags[vm]) {
current[t] = struct{}{}
}
changes := normalizeTagList(tags)
if len(changes) == 0 {
return Cluster{}, errors.New("at least one non-empty tag is required")
}
if add {
for _, t := range changes {
current[t] = struct{}{}
}
} else {
for _, t := range changes {
delete(current, t)
}
}
out := make([]string, 0, len(current))
for t := range current {
out = append(out, t)
}
sort.Strings(out)
if len(out) == 0 {
delete(c.KVMTags, vm)
} else {
c.KVMTags[vm] = out
}
c.KVMTags = normalizeVMTagMap(c.KVMTags)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("kvm.tags", c.Name, vm+"="+strings.Join(out, ","))
return c, nil
}
func (s *Service) ListKVMTags(selector, vm string) (map[string][]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
out := make(map[string][]string, len(c.KVMTags))
for k, v := range c.KVMTags {
if vm != "" && !strings.EqualFold(strings.TrimSpace(vm), k) {
continue
}
out[k] = append([]string(nil), normalizeTagList(v)...)
}
return out, nil
}
internal/cluster/transport.go
+269
View File
@@ -0,0 +1,269 @@
package cluster
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
"golang.org/x/crypto/ssh"
)
// agentClient wraps an *http.Client pointed at the correct base URL for a
// cluster's agent (direct or SSH-tunneled).
type agentClient struct {
http *http.Client
target string
close func()
}
func (a *agentClient) Close() {
if a != nil && a.close != nil {
a.close()
}
}
// newAgentClient builds the right HTTP client for reaching a cluster's agent.
//
// For TransportDirect it returns a plain client talking to cluster.Host:port.
//
// For TransportIPFabric it reuses a cached ssh.Client from the service pool
// and returns a client whose Transport routes every TCP connection through
// ssh.Client.Dial to 127.0.0.1:port. The SSH connection stays pooled after
// Close() — only the HTTP transport's idle conns are released.
func (s *Service) newAgentClient(ctx context.Context, c Cluster, timeout time.Duration) (*agentClient, error) {
if c.Agent.Port == 0 {
return nil, errors.New("agent port is not set")
}
if timeout <= 0 {
timeout = 8 * time.Second
}
scheme := agentScheme(c)
tlsCfg, err := agentTLSConfig(c)
if err != nil {
return nil, err
}
switch normalizeTransport(c.Transport) {
case TransportIPFabric:
sshClient, err := s.acquireTunnelClient(ctx, c)
if err != nil {
return nil, err
}
tr := &http.Transport{
TLSClientConfig: tlsCfg,
DialContext: func(dctx context.Context, network, _ string) (net.Conn, error) {
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(c.Agent.Port))
return sshDialWithContext(dctx, sshClient, network, addr)
},
DisableKeepAlives: true,
IdleConnTimeout: 30 * time.Second,
ResponseHeaderTimeout: timeout,
ExpectContinueTimeout: 1 * time.Second,
}
httpClient := &http.Client{
Timeout: timeout,
Transport: tr,
}
return &agentClient{
http: httpClient,
target: scheme + "://127.0.0.1:" + strconv.Itoa(c.Agent.Port),
close: func() {
tr.CloseIdleConnections()
},
}, nil
default:
tr := &http.Transport{
TLSClientConfig: tlsCfg,
ResponseHeaderTimeout: timeout,
ExpectContinueTimeout: 1 * time.Second,
}
httpClient := &http.Client{Timeout: timeout, Transport: tr}
return &agentClient{
http: httpClient,
target: scheme + "://" + net.JoinHostPort(c.Host, strconv.Itoa(c.Agent.Port)),
close: func() {
tr.CloseIdleConnections()
},
}, nil
}
}
func agentScheme(c Cluster) string {
if c.Agent.TLSEnabled {
return "https"
}
return "http"
}
func agentTLSConfig(c Cluster) (*tls.Config, error) {
if !c.Agent.TLSEnabled {
return nil, nil
}
fp := strings.ToLower(strings.TrimSpace(c.Agent.TLSFingerprint))
if fp == "" {
return nil, errors.New("agent TLS is enabled but certificate fingerprint is missing")
}
fp = strings.ReplaceAll(fp, ":", "")
want, err := hex.DecodeString(fp)
if err != nil {
return nil, fmt.Errorf("invalid agent TLS fingerprint: %w", err)
}
return &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true, // verified via explicit fingerprint pinning below
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("agent TLS: peer certificate is missing")
}
sum := sha256.Sum256(rawCerts[0])
if len(want) != len(sum) {
return errors.New("agent TLS: fingerprint length mismatch")
}
if !hmacEqual(sum[:], want) {
return errors.New("agent TLS: fingerprint mismatch")
}
return nil
},
}, nil
}
func hmacEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
var v byte
for i := 0; i < len(a); i++ {
v |= a[i] ^ b[i]
}
return v == 0
}
// acquireTunnelClient returns a pooled ssh.Client for the cluster, creating
// one if necessary. Credential changes invalidate the cached entry via the
// fingerprint field. Dead clients are evicted lazily: when a DialContext
// through a stale client fails, the caller invokes CloseTunnelClient and the
// next acquire re-dials.
func (s *Service) acquireTunnelClient(ctx context.Context, c Cluster) (*ssh.Client, error) {
fp := credentialFingerprint(c)
s.tunnelMu.Lock()
entry, ok := s.tunnelPool[c.ID]
if ok && entry.fp != fp {
_ = entry.client.Close()
delete(s.tunnelPool, c.ID)
entry = nil
ok = false
}
if ok {
s.tunnelMu.Unlock()
return entry.client, nil
}
s.tunnelMu.Unlock()
client, err := s.dialSSH(ctx, c, "", "")
if err != nil {
return nil, err
}
s.tunnelMu.Lock()
if existing, ok := s.tunnelPool[c.ID]; ok && existing.fp == fp {
// Another goroutine won the race; drop ours.
s.tunnelMu.Unlock()
_ = client.Close()
return existing.client, nil
}
s.tunnelPool[c.ID] = &tunneledSSH{
client: client,
fp: fp,
}
s.tunnelMu.Unlock()
return client, nil
}
// CloseTunnelClient drops a pooled SSH tunnel for a cluster. Safe to call if
// no entry exists.
func (s *Service) CloseTunnelClient(clusterID string) {
s.tunnelMu.Lock()
entry, ok := s.tunnelPool[clusterID]
if ok {
delete(s.tunnelPool, clusterID)
}
s.tunnelMu.Unlock()
if ok && entry != nil && entry.client != nil {
_ = entry.client.Close()
}
}
// CloseAllTunnelClients tears down every pooled SSH tunnel.
func (s *Service) CloseAllTunnelClients() {
s.tunnelMu.Lock()
pool := s.tunnelPool
s.tunnelPool = make(map[string]*tunneledSSH)
s.tunnelMu.Unlock()
for _, e := range pool {
if e != nil && e.client != nil {
_ = e.client.Close()
}
}
}
// credentialFingerprint returns a short hash over the fields that affect how
// we'd reconnect. If any of these change we must not reuse a cached client.
func credentialFingerprint(c Cluster) string {
h := sha256.New()
h.Write([]byte(c.Host))
h.Write([]byte{'|'})
h.Write([]byte(strconv.Itoa(c.Port)))
h.Write([]byte{'|'})
h.Write([]byte(c.User))
h.Write([]byte{'|'})
h.Write([]byte(c.AuthMethod))
h.Write([]byte{'|'})
h.Write([]byte(c.Password))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPath))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPassphrase))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPassphraseFile))
return hex.EncodeToString(h.Sum(nil)[:8])
}
// sshDialWithContext wraps ssh.Client.Dial so it respects ctx cancellation.
// ssh.Client has no context-aware dial, so we fall back to a watcher goroutine
// that closes the connection if ctx fires before the dial returns.
func sshDialWithContext(ctx context.Context, client *ssh.Client, network, addr string) (net.Conn, error) {
type result struct {
conn net.Conn
err error
}
ch := make(chan result, 1)
go func() {
conn, err := client.Dial(network, addr)
ch <- result{conn: conn, err: err}
}()
select {
case <-ctx.Done():
go func() {
r := <-ch
if r.conn != nil {
_ = r.conn.Close()
}
}()
return nil, ctx.Err()
case r := <-ch:
return r.conn, r.err
}
}
internal/cluster/usage.go
+272
View File
@@ -0,0 +1,272 @@
package cluster
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"pxmon/internal/agent"
"pxmon/internal/history"
)
// UsageRange is a façade over history.RangeShortcut so callers outside this
// package don't need to import both.
type UsageRange = history.RangeShortcut
// UsageSnapshot bundles everything the "usage" page shows for a cluster at a
// single point in time.
type UsageSnapshot struct {
ClusterID string `json:"cluster_id"`
ClusterName string `json:"cluster_name"`
Range string `json:"range"`
GeneratedAt time.Time `json:"generated_at"`
Live agent.StatsResponse `json:"live"`
Top agent.TopResponse `json:"top"`
DU agent.DUResponse `json:"du"`
NodeSeries []history.NodeSamplePoint `json:"series,omitempty"`
P95TotalMbps float64 `json:"p95_total_mbps"`
MaxTotalMbps float64 `json:"max_total_mbps"`
AvgTotalMbps float64 `json:"avg_total_mbps"`
TopIfaceName string `json:"top_iface_name,omitempty"`
TopIfaceMbps float64 `json:"top_iface_mbps,omitempty"`
DUError string `json:"du_error,omitempty"`
TopError string `json:"top_error,omitempty"`
HistoryError string `json:"history_error,omitempty"`
}
// AgentTopProcesses calls /api/v1/top on the selected cluster's agent.
func (s *Service) AgentTopProcesses(ctx context.Context, selector string, sampleWindow time.Duration, limit int) (agent.TopResponse, error) {
cluster, err := s.Get(selector)
if err != nil {
return agent.TopResponse{}, err
}
if !cluster.Agent.Installed {
return agent.TopResponse{}, errors.New("agent is not installed on this cluster")
}
ac, err := s.newAgentClient(ctx, cluster, 15*time.Second)
if err != nil {
return agent.TopResponse{}, err
}
defer ac.Close()
url := ac.target + "/api/v1/top"
q := ""
if sampleWindow > 0 {
q += "sample_ms=" + strconv.FormatInt(sampleWindow.Milliseconds(), 10)
}
if limit > 0 {
if q != "" {
q += "&"
}
q += "limit=" + strconv.Itoa(limit)
}
if q != "" {
url += "?" + q
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return agent.TopResponse{}, err
}
applyAgentRequestAuth(req, cluster)
resp, err := ac.http.Do(req)
if err != nil {
if normalizeTransport(cluster.Transport) == TransportIPFabric {
s.CloseTunnelClient(cluster.ID)
}
return agent.TopResponse{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return agent.TopResponse{}, err
}
if resp.StatusCode >= 300 {
msg := string(body)
if len(msg) > 200 {
msg = msg[:200]
}
return agent.TopResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
}
var out agent.TopResponse
if err := json.Unmarshal(body, &out); err != nil {
return agent.TopResponse{}, err
}
return out, nil
}
// AgentDirSizes calls /api/v1/du on the selected cluster's agent.
func (s *Service) AgentDirSizes(ctx context.Context, selector, path string, limit int, timeout time.Duration) (agent.DUResponse, error) {
cluster, err := s.Get(selector)
if err != nil {
return agent.DUResponse{}, err
}
if !cluster.Agent.Installed {
return agent.DUResponse{}, errors.New("agent is not installed on this cluster")
}
httpTimeout := timeout + 10*time.Second
if httpTimeout < 20*time.Second {
httpTimeout = 20 * time.Second
}
ac, err := s.newAgentClient(ctx, cluster, httpTimeout)
if err != nil {
return agent.DUResponse{}, err
}
defer ac.Close()
url := ac.target + "/api/v1/du"
q := ""
if path != "" {
q += "path=" + path
}
if limit > 0 {
if q != "" {
q += "&"
}
q += "limit=" + strconv.Itoa(limit)
}
if timeout > 0 {
if q != "" {
q += "&"
}
q += "timeout_ms=" + strconv.FormatInt(timeout.Milliseconds(), 10)
}
if q != "" {
url += "?" + q
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return agent.DUResponse{}, err
}
applyAgentRequestAuth(req, cluster)
resp, err := ac.http.Do(req)
if err != nil {
if normalizeTransport(cluster.Transport) == TransportIPFabric {
s.CloseTunnelClient(cluster.ID)
}
return agent.DUResponse{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return agent.DUResponse{}, err
}
if resp.StatusCode >= 300 {
msg := string(body)
if len(msg) > 200 {
msg = msg[:200]
}
return agent.DUResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
}
var out agent.DUResponse
if err := json.Unmarshal(body, &out); err != nil {
return agent.DUResponse{}, err
}
return out, nil
}
// CollectUsageSnapshot gathers every data source needed by the usage page:
// live stats, top processes, top folders, and historical aggregate for P95.
// Errors in individual components are recorded on the snapshot instead of
// aborting the whole call — the page degrades gracefully.
func (s *Service) CollectUsageSnapshot(ctx context.Context, selector string, rng UsageRange, duPath string) (UsageSnapshot, error) {
cluster, err := s.Get(selector)
if err != nil {
return UsageSnapshot{}, err
}
snap := UsageSnapshot{
ClusterID: cluster.ID,
ClusterName: cluster.Name,
Range: string(rng),
GeneratedAt: time.Now().UTC(),
}
if live, err := s.AgentStatsTyped(ctx, selector); err != nil {
return snap, fmt.Errorf("live stats: %w", err)
} else {
snap.Live = live
}
if top, err := s.AgentTopProcesses(ctx, selector, 300*time.Millisecond, 15); err != nil {
snap.TopError = err.Error()
} else {
snap.Top = top
}
if duPath == "" {
duPath = "/"
}
if du, err := s.AgentDirSizes(ctx, selector, duPath, 12, 12*time.Second); err != nil {
snap.DUError = err.Error()
} else {
snap.DU = du
}
// Network history + P95 aggregation.
if s.networkStore == nil {
snap.HistoryError = "history store not configured"
} else {
since := history.RangeShortcut(rng).Since(time.Now())
if rng == "" || rng == history.RangeLive {
since = time.Now().Add(-5 * time.Minute)
}
snapshots, err := s.networkStore.Load(cluster.ID, since)
if err != nil {
snap.HistoryError = err.Error()
} else {
// Pick the physical uplink once and use it consistently for
// the series, P95, and the "top iface" display so they all
// describe the same thing.
primary := history.PrimaryInterface(snapshots)
series := history.AggregateNodeSeries(snapshots, primary)
snap.NodeSeries = series
snap.P95TotalMbps = history.PercentileMbps(series, 95)
if len(series) > 0 {
maxV := 0.0
sum := 0.0
for _, p := range series {
if p.TotalMbps > maxV {
maxV = p.TotalMbps
}
sum += p.TotalMbps
}
snap.MaxTotalMbps = maxV
snap.AvgTotalMbps = sum / float64(len(series))
}
snap.TopIfaceName = primary
snap.TopIfaceMbps = snap.AvgTotalMbps
}
}
return snap, nil
}
// RenderUsageChartPNG is a thin helper that reads a usage snapshot's series
// and delegates to history.RenderNodeNetworkPNG.
func RenderUsageChartPNG(snap UsageSnapshot, title string) ([]byte, error) {
if title == "" {
title = fmt.Sprintf("%s — node network usage (%s)", snap.ClusterName, snap.Range)
}
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps",
snap.P95TotalMbps, snap.MaxTotalMbps, snap.AvgTotalMbps)
return history.RenderNodeNetworkPNG(snap.NodeSeries, history.ChartOptions{
Title: title,
Subtitle: subtitle,
Percentile: 95,
})
}
internal/cluster/vm_alerts.go
+137
View File
@@ -0,0 +1,137 @@
package cluster
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
type VMStateSummary struct {
Total int `json:"total"`
Running int `json:"running"`
ShutOff int `json:"shut_off"`
Paused int `json:"paused"`
Others int `json:"others"`
ShutOffNames []string `json:"shut_off_names,omitempty"`
PausedNames []string `json:"paused_names,omitempty"`
OtherNames []string `json:"other_names,omitempty"`
Warnings []string `json:"warnings,omitempty"`
SampledAt time.Time `json:"sampled_at"`
}
func (s *Service) GetVMAlertPolicy(selector string) (VMAlertPolicy, error) {
c, err := s.Get(selector)
if err != nil {
return VMAlertPolicy{}, err
}
return ensureVMAlertPolicy(c.VMAlerts), nil
}
func (s *Service) SetVMAlertPolicy(selector string, p VMAlertPolicy) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.VMAlerts = ensureVMAlertPolicy(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("vm.alerts", c.Name, fmt.Sprintf("enabled=%t warn_on_shutoff=%t min_running=%d", c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning))
return c, nil
}
func (s *Service) CheckVMAlerts(ctx context.Context, selector string) (VMStateSummary, error) {
c, err := s.Get(selector)
if err != nil {
return VMStateSummary{}, err
}
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
if err != nil {
return VMStateSummary{}, err
}
states := parseVirshListStates(out)
summary := VMStateSummary{SampledAt: s.now().UTC(), Total: len(states)}
for name, st := range states {
n := strings.ToLower(strings.TrimSpace(st))
switch {
case n == "running":
summary.Running++
case n == "shut off":
summary.ShutOff++
summary.ShutOffNames = append(summary.ShutOffNames, name)
case n == "paused":
summary.Paused++
summary.PausedNames = append(summary.PausedNames, name)
default:
summary.Others++
summary.OtherNames = append(summary.OtherNames, name)
}
}
sort.Strings(summary.ShutOffNames)
sort.Strings(summary.PausedNames)
sort.Strings(summary.OtherNames)
p := ensureVMAlertPolicy(c.VMAlerts)
if p.Enabled {
if p.WarnOnShutoff && summary.ShutOff > 0 {
summary.Warnings = append(summary.Warnings, fmt.Sprintf(
"%d VM(s) are shut off: %s",
summary.ShutOff,
joinNamesLimit(summary.ShutOffNames, 12),
))
}
if summary.Running < p.MinRunning {
summary.Warnings = append(summary.Warnings, fmt.Sprintf("running VM count %d is below min_running=%d", summary.Running, p.MinRunning))
}
}
sort.Strings(summary.Warnings)
return summary, nil
}
func (s *Service) ListVMStates(ctx context.Context, selector string) (map[string]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
if err != nil {
return nil, err
}
return parseVirshListStates(out), nil
}
func parseVirshListStates(raw string) map[string]string {
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
out := make(map[string]string)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "Id") || strings.HasPrefix(line, "-") {
continue
}
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
name := fields[1]
state := strings.Join(fields[2:], " ")
out[name] = state
}
return out
}
func joinNamesLimit(items []string, limit int) string {
if len(items) == 0 {
return "-"
}
if limit <= 0 || len(items) <= limit {
return strings.Join(items, ", ")
}
return strings.Join(items[:limit], ", ") + fmt.Sprintf(" (+%d more)", len(items)-limit)
}
internal/history/availability_store.go
+92
View File
@@ -0,0 +1,92 @@
package history
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
type AvailabilitySnapshot struct {
Timestamp time.Time `json:"timestamp"`
ClusterUp bool `json:"cluster_up"`
VMStates map[string]string `json:"vm_states,omitempty"`
Error string `json:"error,omitempty"`
}
type AvailabilityStore struct {
dir string
}
func NewAvailabilityStore(baseDir string) *AvailabilityStore {
if strings.TrimSpace(baseDir) == "" {
baseDir = "."
}
return &AvailabilityStore{dir: filepath.Join(baseDir, "history", "availability")}
}
func (s *AvailabilityStore) Path(clusterID string) string {
return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
}
func (s *AvailabilityStore) Append(clusterID string, snap AvailabilitySnapshot) error {
if strings.TrimSpace(clusterID) == "" {
return errors.New("empty cluster id")
}
if snap.Timestamp.IsZero() {
snap.Timestamp = time.Now().UTC()
}
if err := os.MkdirAll(s.dir, 0o700); err != nil {
return err
}
f, err := os.OpenFile(s.Path(clusterID), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer f.Close()
b, err := json.Marshal(snap)
if err != nil {
return err
}
_, err = f.Write(append(b, '\n'))
return err
}
func (s *AvailabilityStore) Load(clusterID string, since time.Time) ([]AvailabilitySnapshot, error) {
if strings.TrimSpace(clusterID) == "" {
return nil, errors.New("empty cluster id")
}
f, err := os.Open(s.Path(clusterID))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return []AvailabilitySnapshot{}, nil
}
return nil, err
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
out := make([]AvailabilitySnapshot, 0, 512)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var snap AvailabilitySnapshot
if err := json.Unmarshal([]byte(line), &snap); err != nil {
continue
}
if !since.IsZero() && snap.Timestamp.Before(since) {
continue
}
out = append(out, snap)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("scan availability: %w", err)
}
return out, nil
}
internal/history/capacity_store.go
+102
View File
@@ -0,0 +1,102 @@
package history
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
type CapacityDiskPoint struct {
Mount string `json:"mount"`
UsedBytes uint64 `json:"used_bytes"`
TotalBytes uint64 `json:"total_bytes"`
}
type CapacitySnapshot struct {
Timestamp time.Time `json:"timestamp"`
Disks []CapacityDiskPoint `json:"disks,omitempty"`
}
type CapacityStore struct {
dir string
}
func NewCapacityStore(baseDir string) *CapacityStore {
if strings.TrimSpace(baseDir) == "" {
baseDir = "."
}
return &CapacityStore{dir: filepath.Join(baseDir, "history", "capacity")}
}
func (s *CapacityStore) Path(clusterID string) string {
return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
}
func (s *CapacityStore) Append(clusterID string, snap CapacitySnapshot) error {
if strings.TrimSpace(clusterID) == "" {
return errors.New("empty cluster id")
}
if snap.Timestamp.IsZero() {
snap.Timestamp = time.Now().UTC()
}
if len(snap.Disks) == 0 {
return nil
}
if err := os.MkdirAll(s.dir, 0o700); err != nil {
return err
}
f, err := os.OpenFile(s.Path(clusterID), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer f.Close()
b, err := json.Marshal(snap)
if err != nil {
return err
}
_, err = f.Write(append(b, '\n'))
return err
}
func (s *CapacityStore) Load(clusterID string, since time.Time) ([]CapacitySnapshot, error) {
if strings.TrimSpace(clusterID) == "" {
return nil, errors.New("empty cluster id")
}
f, err := os.Open(s.Path(clusterID))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return []CapacitySnapshot{}, nil
}
return nil, err
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
out := make([]CapacitySnapshot, 0, 512)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var snap CapacitySnapshot
if err := json.Unmarshal([]byte(line), &snap); err != nil {
continue
}
if !since.IsZero() && snap.Timestamp.Before(since) {
continue
}
if len(snap.Disks) == 0 {
continue
}
out = append(out, snap)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("scan capacity: %w", err)
}
return out, nil
}
internal/history/chart.go
+199
View File
@@ -0,0 +1,199 @@
package history
import (
"bytes"
"fmt"
"time"
chart "github.com/wcharczuk/go-chart/v2"
"github.com/wcharczuk/go-chart/v2/drawing"
)
// ChartOptions controls rendering of the node network PNG.
type ChartOptions struct {
Title string
Subtitle string
Width int
Height int
Percentile float64
}
// RenderNodeNetworkPNG returns a PNG of the node-wide Rx+Tx time series with
// the requested percentile drawn as a horizontal annotation line.
func RenderNodeNetworkPNG(points []NodeSamplePoint, opts ChartOptions) ([]byte, error) {
if opts.Width <= 0 {
opts.Width = 1280
}
if opts.Height <= 0 {
opts.Height = 640
}
if opts.Percentile <= 0 {
opts.Percentile = 95
}
if opts.Title == "" {
opts.Title = "Node network usage"
}
if len(points) < 2 {
return renderEmptyChart(opts, "insufficient history — need at least 2 samples")
}
xs := make([]time.Time, 0, len(points))
rxs := make([]float64, 0, len(points))
txs := make([]float64, 0, len(points))
totals := make([]float64, 0, len(points))
maxVal := 0.0
for _, p := range points {
xs = append(xs, p.Timestamp)
rxs = append(rxs, p.RxMbps)
txs = append(txs, p.TxMbps)
totals = append(totals, p.TotalMbps)
if p.TotalMbps > maxVal {
maxVal = p.TotalMbps
}
}
pct := PercentileMbps(points, opts.Percentile)
yMax := maxVal * 1.15
if pct*1.10 > yMax {
yMax = pct * 1.10
}
if yMax <= 0 {
yMax = 1
}
percentileSeries := chart.ContinuousSeries{
Name: fmt.Sprintf("P%.0f = %.1f Mbps", opts.Percentile, pct),
Style: chart.Style{
StrokeColor: drawing.ColorFromHex("e74c3c"),
StrokeWidth: 2.0,
StrokeDashArray: []float64{6, 4},
},
XValues: []float64{chart.TimeToFloat64(xs[0]), chart.TimeToFloat64(xs[len(xs)-1])},
YValues: []float64{pct, pct},
}
graph := chart.Chart{
Title: opts.Title,
TitleStyle: chart.Style{
FontSize: 16,
},
Width: opts.Width,
Height: opts.Height,
Background: chart.Style{
Padding: chart.Box{Top: 60, Left: 40, Right: 40, Bottom: 40},
FillColor: drawing.Color{
R: 0xf8, G: 0xf9, B: 0xfa, A: 0xff,
},
},
XAxis: chart.XAxis{
Style: chart.Style{FontSize: 9},
ValueFormatter: chart.TimeValueFormatterWithFormat("15:04:05\n02 Jan"),
},
YAxis: chart.YAxis{
Name: "Mbps",
Style: chart.Style{FontSize: 9},
Range: &chart.ContinuousRange{Min: 0, Max: yMax},
ValueFormatter: func(v any) string {
if f, ok := v.(float64); ok {
return formatMbps(f)
}
return ""
},
},
Series: []chart.Series{
chart.TimeSeries{
Name: "Total Rx+Tx",
Style: chart.Style{
StrokeColor: drawing.ColorFromHex("2d7dd2"),
StrokeWidth: 2.0,
FillColor: drawing.ColorFromHex("2d7dd2").WithAlpha(50),
},
XValues: xs,
YValues: totals,
},
chart.TimeSeries{
Name: "Rx",
Style: chart.Style{
StrokeColor: drawing.ColorFromHex("3cb371"),
StrokeWidth: 1.5,
},
XValues: xs,
YValues: rxs,
},
chart.TimeSeries{
Name: "Tx",
Style: chart.Style{
StrokeColor: drawing.ColorFromHex("ffa500"),
StrokeWidth: 1.5,
},
XValues: xs,
YValues: txs,
},
percentileSeries,
},
}
if opts.Subtitle != "" {
graph.Elements = []chart.Renderable{subtitleRenderable(opts.Subtitle)}
}
buf := &bytes.Buffer{}
if err := graph.Render(chart.PNG, buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func subtitleRenderable(text string) chart.Renderable {
return func(r chart.Renderer, cb chart.Box, chartDefaults chart.Style) {
r.SetFont(chartDefaults.GetFont())
r.SetFontColor(drawing.Color{R: 90, G: 90, B: 90, A: 0xff})
r.SetFontSize(10)
r.Text(text, cb.Left+10, cb.Top+30)
}
}
func renderEmptyChart(opts ChartOptions, note string) ([]byte, error) {
graph := chart.Chart{
Title: opts.Title,
Width: opts.Width,
Height: opts.Height,
Background: chart.Style{
Padding: chart.Box{Top: 60, Left: 40, Right: 40, Bottom: 40},
},
Series: []chart.Series{
chart.ContinuousSeries{
XValues: []float64{0, 1},
YValues: []float64{0, 0},
Style: chart.Style{
StrokeColor: drawing.ColorTransparent,
},
},
},
}
graph.Elements = []chart.Renderable{
func(r chart.Renderer, cb chart.Box, cs chart.Style) {
r.SetFont(cs.GetFont())
r.SetFontColor(drawing.Color{R: 120, G: 120, B: 120, A: 0xff})
r.SetFontSize(14)
r.Text(note, cb.Left+20, cb.Top+cb.Height()/2)
},
}
buf := &bytes.Buffer{}
if err := graph.Render(chart.PNG, buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func formatMbps(v float64) string {
switch {
case v >= 1000:
return fmt.Sprintf("%.2f Gbps", v/1000)
case v >= 1:
return fmt.Sprintf("%.1f Mbps", v)
default:
return fmt.Sprintf("%.0f Kbps", v*1000)
}
}
internal/history/network_store.go
+146
View File
@@ -0,0 +1,146 @@
package history
import (
"bufio"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
// InterfaceSample is one per-interface network sample at timestamp.
type InterfaceSample struct {
Interface string `json:"interface"`
RxMbps float64 `json:"rx_mbps"`
TxMbps float64 `json:"tx_mbps"`
RxDrops uint64 `json:"rx_drops"`
TxDrops uint64 `json:"tx_drops"`
}
// NetworkSnapshot stores one full sample containing many interfaces.
type NetworkSnapshot struct {
Timestamp time.Time `json:"timestamp"`
Interfaces []InterfaceSample `json:"interfaces"`
}
// NetworkStore appends and reads network snapshots (JSONL) per cluster.
type NetworkStore struct {
dir string
}
func NewNetworkStore(baseDir string) *NetworkStore {
if strings.TrimSpace(baseDir) == "" {
baseDir = "."
}
return &NetworkStore{
dir: filepath.Join(baseDir, "history", "network"),
}
}
func (s *NetworkStore) Path(clusterID string) string {
return filepath.Join(s.dir, sanitizeClusterID(clusterID)+".jsonl")
}
func (s *NetworkStore) Append(clusterID string, snapshot NetworkSnapshot) error {
if strings.TrimSpace(clusterID) == "" {
return errors.New("empty cluster id")
}
if snapshot.Timestamp.IsZero() {
snapshot.Timestamp = time.Now().UTC()
}
if len(snapshot.Interfaces) == 0 {
return nil
}
if err := os.MkdirAll(s.dir, 0o700); err != nil {
return fmt.Errorf("create history dir: %w", err)
}
path := s.Path(clusterID)
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return fmt.Errorf("open history file: %w", err)
}
defer f.Close()
line, err := json.Marshal(snapshot)
if err != nil {
return fmt.Errorf("marshal history snapshot: %w", err)
}
if _, err := f.Write(append(line, '\n')); err != nil {
return fmt.Errorf("append history snapshot: %w", err)
}
return nil
}
func (s *NetworkStore) Load(clusterID string, since time.Time) ([]NetworkSnapshot, error) {
if strings.TrimSpace(clusterID) == "" {
return nil, errors.New("empty cluster id")
}
path := s.Path(clusterID)
f, err := os.Open(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return []NetworkSnapshot{}, nil
}
return nil, fmt.Errorf("open history file: %w", err)
}
defer f.Close()
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 1024*64), 1024*1024*8)
out := make([]NetworkSnapshot, 0, 256)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var snap NetworkSnapshot
if err := json.Unmarshal([]byte(line), &snap); err != nil {
continue
}
if !since.IsZero() && snap.Timestamp.Before(since) {
continue
}
if len(snap.Interfaces) == 0 {
continue
}
out = append(out, snap)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("scan history file: %w", err)
}
return out, nil
}
func sanitizeClusterID(v string) string {
v = strings.TrimSpace(v)
if v == "" {
return "unknown"
}
var b strings.Builder
for _, r := range v {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
out := strings.Trim(b.String(), "_")
if out == "" {
return "unknown"
}
return out
}
internal/history/percentile.go
+242
View File
@@ -0,0 +1,242 @@
package history
import (
"math"
"sort"
"strings"
"time"
)
// NodeSamplePoint is one aggregated sample for the node's primary uplink
// interface at a single point in time. TotalMbps uses max(Rx, Tx) which is
// the same convention a provider uses for 95th-percentile transit billing.
type NodeSamplePoint struct {
Timestamp time.Time
TotalMbps float64 // max(Rx, Tx) for billing P95
RxMbps float64
TxMbps float64
Interface string
}
// isVirtualIface returns true for interfaces that do not represent real
// uplink traffic and should be excluded from P95/billing calculations.
// Virtual interfaces (bridges, taps, veth pairs, docker, loopback) either
// carry no real traffic or mirror the traffic that already flows through
// the physical uplink — double-counting them inflates totals.
func isVirtualIface(name string) bool {
n := strings.ToLower(name)
if n == "lo" || n == "" {
return true
}
prefixes := []string{
"lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr",
"cni", "flannel", "wg", "tun", "tailscale", "zt", "ipsec",
"kube", "cilium", "ovs", "podman", "dummy",
}
for _, p := range prefixes {
if strings.HasPrefix(n, p) {
return true
}
}
return false
}
// PrimaryInterface picks the physical uplink interface with the highest
// average max(Rx, Tx) over the given snapshots. Virtual interfaces are
// ignored. Returns empty string when no physical candidate exists.
func PrimaryInterface(snapshots []NetworkSnapshot) string {
type acc struct {
sum float64
count int
}
totals := make(map[string]*acc, 16)
for _, snap := range snapshots {
for _, iface := range snap.Interfaces {
if isVirtualIface(iface.Interface) {
continue
}
v := math.Max(iface.RxMbps, iface.TxMbps)
a, ok := totals[iface.Interface]
if !ok {
a = &acc{}
totals[iface.Interface] = a
}
a.sum += v
a.count++
}
}
best := ""
bestAvg := -1.0
for name, a := range totals {
if a.count == 0 {
continue
}
avg := a.sum / float64(a.count)
if avg > bestAvg {
bestAvg = avg
best = name
}
}
return best
}
// AggregateNodeSeries builds a time series for a single interface. If
// ifaceName is empty, PrimaryInterface is used to auto-select the physical
// uplink. TotalMbps uses max(Rx, Tx), matching provider billing convention.
func AggregateNodeSeries(snapshots []NetworkSnapshot, ifaceName string) []NodeSamplePoint {
if ifaceName == "" {
ifaceName = PrimaryInterface(snapshots)
}
out := make([]NodeSamplePoint, 0, len(snapshots))
for _, snap := range snapshots {
for _, iface := range snap.Interfaces {
if iface.Interface != ifaceName {
continue
}
rx := iface.RxMbps
tx := iface.TxMbps
out = append(out, NodeSamplePoint{
Timestamp: snap.Timestamp,
Interface: ifaceName,
RxMbps: rx,
TxMbps: tx,
TotalMbps: math.Max(rx, tx),
})
break
}
}
sort.Slice(out, func(i, j int) bool {
return out[i].Timestamp.Before(out[j].Timestamp)
})
return out
}
// PercentileMbps returns the given percentile (0..100) of the TotalMbps
// field across the series using nearest-rank (inclusive) computation.
// Returns 0 if the series is empty.
func PercentileMbps(points []NodeSamplePoint, percentile float64) float64 {
if len(points) == 0 {
return 0
}
if percentile < 0 {
percentile = 0
}
if percentile > 100 {
percentile = 100
}
values := make([]float64, 0, len(points))
for _, p := range points {
values = append(values, p.TotalMbps)
}
sort.Float64s(values)
if len(values) == 1 {
return values[0]
}
rank := (percentile / 100.0) * float64(len(values)-1)
lo := int(math.Floor(rank))
hi := int(math.Ceil(rank))
if lo == hi {
return values[lo]
}
frac := rank - float64(lo)
return values[lo]*(1-frac) + values[hi]*frac
}
// TopInterfaceByTraffic returns the physical interface with the highest
// average max(Rx, Tx) across the sampled period, together with that
// average throughput in Mbps. Virtual interfaces are ignored.
func TopInterfaceByTraffic(snapshots []NetworkSnapshot) (string, float64) {
type acc struct {
sum float64
count int
}
totals := make(map[string]*acc, 16)
for _, snap := range snapshots {
for _, iface := range snap.Interfaces {
if isVirtualIface(iface.Interface) {
continue
}
v := math.Max(iface.RxMbps, iface.TxMbps)
a, ok := totals[iface.Interface]
if !ok {
a = &acc{}
totals[iface.Interface] = a
}
a.sum += v
a.count++
}
}
name := ""
bestAvg := 0.0
for k, a := range totals {
if a.count == 0 {
continue
}
avg := a.sum / float64(a.count)
if avg > bestAvg {
bestAvg = avg
name = k
}
}
return name, bestAvg
}
// RangeShortcut is a common time-window selector.
type RangeShortcut string
const (
RangeLive RangeShortcut = "live"
RangeHour RangeShortcut = "1h"
RangeDay RangeShortcut = "1d"
RangeMonth RangeShortcut = "1mo"
RangeAll RangeShortcut = "all"
)
// Since returns an absolute start time for the given range shortcut,
// relative to now. RangeAll and RangeLive return zero (no lower bound).
func (r RangeShortcut) Since(now time.Time) time.Time {
switch r {
case RangeHour:
return now.Add(-time.Hour)
case RangeDay:
return now.Add(-24 * time.Hour)
case RangeMonth:
return now.Add(-30 * 24 * time.Hour)
default:
return time.Time{}
}
}
// ParseRangeShortcut accepts user-provided range strings.
func ParseRangeShortcut(raw string) (RangeShortcut, bool) {
switch raw {
case "live", "now":
return RangeLive, true
case "1h", "hour":
return RangeHour, true
case "1d", "day", "24h":
return RangeDay, true
case "1mo", "30d", "month":
return RangeMonth, true
case "all", "":
return RangeAll, true
}
return "", false
}
// Label returns a human-friendly label for the range.
func (r RangeShortcut) Label() string {
switch r {
case RangeLive:
return "live"
case RangeHour:
return "last 1h"
case RangeDay:
return "last 24h"
case RangeMonth:
return "last 30d"
case RangeAll:
return "all time"
}
return string(r)
}