chore: publish pxmon v0.2.0

This commit is contained in:
2026-06-16 21:52:10 +04:00
commit 6b703db02b
69 changed files with 25886 additions and 0 deletions
+42
View File
@@ -0,0 +1,42 @@
package cluster
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"net/http"
"strconv"
"strings"
"time"
)
const (
agentHeaderTS = "X-Observer-Ts"
agentHeaderNonce = "X-Observer-Nonce"
agentHeaderSignature = "X-Observer-Signature"
)
func applyAgentRequestAuth(req *http.Request, c Cluster) {
if req == nil {
return
}
if strings.TrimSpace(c.Agent.Token) != "" {
req.Header.Set("Authorization", "Bearer "+c.Agent.Token)
}
secret := strings.TrimSpace(c.Agent.RequestSecret)
if secret == "" {
return
}
ts := strconv.FormatInt(time.Now().UTC().Unix(), 10)
nonce := randomHex(12)
payload := req.Method + "\n" + req.URL.RequestURI() + "\n" + ts + "\n" + nonce
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(payload))
sig := hex.EncodeToString(mac.Sum(nil))
req.Header.Set(agentHeaderTS, ts)
req.Header.Set(agentHeaderNonce, nonce)
req.Header.Set(agentHeaderSignature, sig)
}
+68
View File
@@ -0,0 +1,68 @@
package cluster
import (
"embed"
"encoding/json"
"strings"
)
//go:embed agent_versions.json
var agentVersionsFS embed.FS
type AgentVersionInfo struct {
Version string `json:"version"`
ReleasedAt string `json:"released_at,omitempty"`
Features []string `json:"features,omitempty"`
}
func loadAgentVersions() []AgentVersionInfo {
raw, err := agentVersionsFS.ReadFile("agent_versions.json")
if err != nil {
return nil
}
var items []AgentVersionInfo
if err := json.Unmarshal(raw, &items); err != nil {
return nil
}
out := make([]AgentVersionInfo, 0, len(items))
for _, it := range items {
it.Version = strings.TrimSpace(it.Version)
if it.Version == "" {
continue
}
out = append(out, it)
}
return out
}
func (s *Service) AgentVersions() []AgentVersionInfo {
return append([]AgentVersionInfo(nil), loadAgentVersions()...)
}
func (s *Service) AgentVersionFeatures(version string) []string {
v := strings.TrimSpace(version)
if v == "" {
return nil
}
for _, it := range loadAgentVersions() {
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
return append([]string(nil), it.Features...)
}
}
return nil
}
func (s *Service) CompareAgentVersion(version string) (isLatest bool, latest string, known bool) {
items := loadAgentVersions()
if len(items) == 0 {
return true, "", false
}
latest = strings.TrimSpace(items[len(items)-1].Version)
v := strings.TrimSpace(version)
for _, it := range items {
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
return strings.EqualFold(v, latest), latest, true
}
}
return false, latest, false
}
+37
View File
@@ -0,0 +1,37 @@
[
{
"version": "dev",
"released_at": "2026-04-10",
"features": [
"cluster usage/traffic/graph",
"kvm top static spec",
"telegram graph sendPhoto fallback"
]
},
{
"version": "dev-2026.04.17",
"released_at": "2026-04-17",
"features": [
"cluster p95 by interface",
"cluster tag + kvm tag",
"cluster vm alert-rules",
"cluster drift",
"cluster runbook",
"cluster scheduler",
"cluster change-history",
"cluster report export"
]
},
{
"version": "v0.2.0",
"released_at": "2026-06-16",
"features": [
"repo tunneling gateway/proxy workflows",
"cluster exec/run commands",
"ssh key passphrase file support",
"export/import referenced key files",
"signed agent request skew hardening",
"tui refresh and network dashboard improvements"
]
}
]
+655
View File
@@ -0,0 +1,655 @@
package cluster
import (
"context"
"fmt"
"io"
"net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type BackupRunResult struct {
PlanID string `json:"plan_id"`
PlanName string `json:"plan_name"`
TargetID string `json:"target_id"`
TargetName string `json:"target_name"`
ArchiveName string `json:"archive_name"`
UploadedTo string `json:"uploaded_to"`
SizeBytes int64 `json:"size_bytes"`
RanAt time.Time `json:"ran_at"`
}
func (s *Service) BackupListTargets() ([]BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil
}
func (s *Service) BackupListPlans() ([]BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil
}
func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) {
reg, err := s.store.Load()
if err != nil {
return "", err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
var target *BackupTarget
for i := range cfg.Targets {
if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return "", fmt.Errorf("backup target not found")
}
switch target.Type {
case "sftp":
host := strings.TrimSpace(target.SFTPHost)
user := strings.TrimSpace(target.SFTPUser)
if host == "" || user == "" {
return "", fmt.Errorf("sftp target has empty host/user")
}
addr := netJoinHostPort(host, target.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(target.SFTPPassword) != "" {
auths = append(auths, ssh.Password(target.SFTPPassword))
}
if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(target.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
if _, err := c.ReadDir(base); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil
case "s3":
endpoint := strings.TrimSpace(target.S3Endpoint)
bucket := strings.TrimSpace(target.S3Bucket)
access := strings.TrimSpace(target.S3AccessKey)
secret := strings.TrimSpace(target.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(target.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if target.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: target.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
exists, err := cli.BucketExists(ctx, bucket)
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket)
}
scheme := "https"
if !target.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket, nil
default:
return "", fmt.Errorf("unsupported target type %q", target.Type)
}
}
func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
t.ID = strings.TrimSpace(t.ID)
if t.ID == "" {
t.ID = newClusterID()
}
t.Name = strings.TrimSpace(t.Name)
if t.Name == "" {
return BackupTarget{}, fmt.Errorf("target name is required")
}
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
if t.Type != "sftp" && t.Type != "s3" {
return BackupTarget{}, fmt.Errorf("target type must be sftp|s3")
}
if t.Type == "sftp" {
if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user")
}
if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires password or key")
}
}
if t.Type == "s3" {
if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket")
}
if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key")
}
}
for _, ex := range cfg.Targets {
if strings.EqualFold(ex.Name, t.Name) {
return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name)
}
}
now := s.now().UTC()
t.CreatedAt = now
t.UpdatedAt = now
if t.SFTPPort <= 0 {
t.SFTPPort = 22
}
if !t.Enabled {
t.Enabled = true
}
cfg.Targets = append(cfg.Targets, t)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.add", t.ID, t.Name)
return t, nil
}
func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, t := range cfg.Targets {
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupTarget{}, fmt.Errorf("backup target not found")
}
removed := cfg.Targets[idx]
cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
if strings.TrimSpace(p.Name) == "" {
return BackupPlan{}, fmt.Errorf("plan name is required")
}
if strings.TrimSpace(p.TargetID) == "" {
return BackupPlan{}, fmt.Errorf("target is required")
}
if len(p.Paths) == 0 {
return BackupPlan{}, fmt.Errorf("at least one path is required")
}
targetID := ""
for _, t := range cfg.Targets {
if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) {
targetID = t.ID
break
}
}
if targetID == "" {
return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID)
}
for _, ex := range cfg.Plans {
if strings.EqualFold(ex.Name, p.Name) {
return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name)
}
}
p.ID = newClusterID()
p.TargetID = targetID
p.Paths = normalizeBackupPaths(p.Paths)
if strings.TrimSpace(p.Every) == "" {
p.Every = "24h"
}
if p.RetainDays <= 0 {
p.RetainDays = 30
}
p.Compress = true
now := s.now().UTC()
p.CreatedAt = now
p.UpdatedAt = now
if !p.Enabled {
p.Enabled = true
}
cfg.Plans = append(cfg.Plans, p)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.add", p.ID, p.Name)
return p, nil
}
func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, p := range cfg.Plans {
if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupPlan{}, fmt.Errorf("backup plan not found")
}
removed := cfg.Plans[idx]
cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) {
reg, err := s.store.Load()
if err != nil {
return BackupRunResult{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
var plan *BackupPlan
for i := range cfg.Plans {
if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) {
plan = &cfg.Plans[i]
break
}
}
if plan == nil {
return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector)
}
var target *BackupTarget
for i := range cfg.Targets {
if cfg.Targets[i].ID == plan.TargetID {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID)
}
c, err := s.Get(plan.Cluster)
if err != nil {
return BackupRunResult{}, err
}
sshClient, err := s.dialSSH(ctx, c, "", "")
if err != nil {
return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err)
}
defer sshClient.Close()
ts := s.now().UTC().Format("20060102T150405Z")
archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz"
tmpPath := filepath.Join(os.TempDir(), archiveName)
tmpFile, err := os.Create(tmpPath)
if err != nil {
return BackupRunResult{}, err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpPath)
}()
remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths)
if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err)
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return BackupRunResult{}, err
}
st, _ := tmpFile.Stat()
size := int64(0)
if st != nil {
size = st.Size()
}
uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size)
if err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("upload backup: %w", err)
}
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "ok"
plan.LastError = ""
plan.LastArchive = archiveName
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupRunResult{}, err
}
_ = s.AppendChange("backup.plan.run", plan.ID, archiveName)
return BackupRunResult{
PlanID: plan.ID,
PlanName: plan.Name,
TargetID: target.ID,
TargetName: target.Name,
ArchiveName: archiveName,
UploadedTo: uploadedTo,
SizeBytes: size,
RanAt: plan.LastRunAt,
}, nil
}
func normalizeBackupPaths(in []string) []string {
out := make([]string, 0, len(in))
seen := map[string]struct{}{}
for _, p := range in {
v := strings.TrimSpace(p)
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
return out
}
func sanitizeBackupName(v string) string {
v = strings.ToLower(strings.TrimSpace(v))
if v == "" {
return "backup"
}
var b strings.Builder
for _, r := range v {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
return "backup"
}
return out
}
func buildRemoteTarStreamCommand(paths []string) string {
items := make([]string, 0, len(paths))
for _, p := range paths {
v := strings.TrimSpace(p)
if v == "" {
continue
}
items = append(items, shellQuote(v))
}
if len(items) == 0 {
items = []string{shellQuote("/")}
}
return "tar -czf - " + strings.Join(items, " ")
}
func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error {
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stdout, err := session.StdoutPipe()
if err != nil {
return err
}
stderr, err := session.StderrPipe()
if err != nil {
return err
}
if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
return err
}
done := make(chan error, 1)
go func() {
_, cpErr := io.Copy(w, stdout)
if cpErr != nil {
done <- cpErr
return
}
done <- session.Wait()
}()
select {
case <-ctx.Done():
_ = session.Close()
return ctx.Err()
case err := <-done:
if err == nil {
return nil
}
b, _ := io.ReadAll(stderr)
msg := strings.TrimSpace(string(b))
if msg == "" {
return err
}
return fmt.Errorf("%w: %s", err, msg)
}
}
func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
switch strings.ToLower(strings.TrimSpace(target.Type)) {
case "sftp":
return uploadBackupSFTP(ctx, target, archiveName, r)
case "s3":
return uploadBackupS3(ctx, target, archiveName, r, size)
default:
return "", fmt.Errorf("unsupported backup target type %q", target.Type)
}
}
func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) {
_ = ctx
host := strings.TrimSpace(t.SFTPHost)
if host == "" {
return "", fmt.Errorf("sftp_host is required")
}
user := strings.TrimSpace(t.SFTPUser)
if user == "" {
return "", fmt.Errorf("sftp_user is required")
}
addr := netJoinHostPort(host, t.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(t.SFTPPassword) != "" {
auths = append(auths, ssh.Password(t.SFTPPassword))
}
if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(t.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
remote := path.Join(base, archiveName)
f, err := c.Create(remote)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(f, r); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil
}
func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
endpoint := strings.TrimSpace(t.S3Endpoint)
bucket := strings.TrimSpace(t.S3Bucket)
access := strings.TrimSpace(t.S3AccessKey)
secret := strings.TrimSpace(t.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(t.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if t.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: t.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
key := archiveName
if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" {
key = p + "/" + archiveName
}
opts := minio.PutObjectOptions{ContentType: "application/gzip"}
if size < 0 {
size = -1
}
_, err = cli.PutObject(ctx, bucket, key, r, size, opts)
if err != nil {
return "", err
}
scheme := "https"
if !t.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil
}
func netJoinHostPort(host string, port int) string {
p := port
if p <= 0 {
p = 22
}
return net.JoinHostPort(host, strconv.Itoa(p))
}
+89
View File
@@ -0,0 +1,89 @@
package cluster
import (
"bufio"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
)
type ChangeRecord struct {
At time.Time `json:"at"`
Action string `json:"action"`
Target string `json:"target"`
Details string `json:"details,omitempty"`
}
func (s *Service) changeHistoryPath() string {
return filepath.Join(s.DataDir(), "history", "changes.log")
}
func (s *Service) AppendChange(action, target, details string) error {
if s == nil {
return nil
}
rec := ChangeRecord{
At: s.now().UTC(),
Action: strings.TrimSpace(action),
Target: strings.TrimSpace(target),
Details: strings.TrimSpace(details),
}
if rec.Action == "" {
return nil
}
path := s.changeHistoryPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return err
}
defer f.Close()
b, err := json.Marshal(rec)
if err != nil {
return err
}
_, err = f.Write(append(b, '\n'))
return err
}
func (s *Service) ListChanges(limit int) ([]ChangeRecord, error) {
path := s.changeHistoryPath()
f, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
defer f.Close()
capHint := 32
if limit > capHint {
capHint = limit
}
items := make([]ChangeRecord, 0, capHint)
sc := bufio.NewScanner(f)
sc.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" {
continue
}
var rec ChangeRecord
if err := json.Unmarshal([]byte(line), &rec); err != nil {
continue
}
items = append(items, rec)
}
if err := sc.Err(); err != nil {
return nil, fmt.Errorf("scan change history: %w", err)
}
if limit > 0 && len(items) > limit {
items = items[len(items)-limit:]
}
return items, nil
}
+86
View File
@@ -0,0 +1,86 @@
package cluster
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
type DriftIssue struct {
Level string `json:"level"`
Kind string `json:"kind"`
Message string `json:"message"`
}
type DriftReport struct {
Cluster string `json:"cluster"`
Generated time.Time `json:"generated_at"`
Issues []DriftIssue `json:"issues,omitempty"`
}
func (s *Service) DetectDrift(ctx context.Context, selector string) (DriftReport, error) {
c, err := s.Get(selector)
if err != nil {
return DriftReport{}, err
}
rep := DriftReport{Cluster: c.Name, Generated: s.now().UTC()}
expected := strings.TrimSpace(s.ExpectedAgentVersion())
dctrl := normalizeDriftControl(c.Drift)
if !c.Agent.Installed {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent", Message: "agent is not installed"})
return rep, nil
}
pingCtx, cancel := context.WithTimeout(ctx, 2500*time.Millisecond)
ping, pingErr := s.PingAgent(pingCtx, c.ID)
cancel()
if pingErr != nil || !ping.Reachable || ping.StatusCode >= 400 {
rep.Issues = append(rep.Issues, DriftIssue{Level: "crit", Kind: "agent", Message: "agent is unreachable"})
} else {
nodeVersion := strings.TrimSpace(ping.Version)
if nodeVersion == "" {
nodeVersion = strings.TrimSpace(c.Agent.Version)
}
if expected != "" && nodeVersion != "" && nodeVersion != expected {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_version", Message: fmt.Sprintf("agent version mismatch: node=%s local=%s", nodeVersion, expected)})
}
if nodeVersion != "" {
if ok, latest, known := s.CompareAgentVersion(nodeVersion); known && !ok {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_outdated", Message: fmt.Sprintf("node version %s is older than latest known %s", nodeVersion, latest)})
}
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.AgentVersion) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.AgentVersion), nodeVersion) {
rep.Issues = append(rep.Issues, DriftIssue{
Level: "warn",
Kind: "baseline_agent_version",
Message: fmt.Sprintf("baseline agent version mismatch: baseline=%s live=%s", dctrl.Baseline.AgentVersion, nodeVersion),
})
}
}
}
softCtx, softCancel := context.WithTimeout(ctx, 7*time.Second)
fresh, softErr := s.probeSoftware(softCtx, c, "", "")
softCancel()
if softErr != nil {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software_probe", Message: softErr.Error()})
} else {
oldSet := strings.TrimSpace(c.Software.Summary())
newSet := strings.TrimSpace(fresh.Summary())
if oldSet != "" && oldSet != "-" && newSet != oldSet {
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software", Message: fmt.Sprintf("software support changed: stored=%s live=%s", oldSet, newSet)})
}
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.Software) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.Software), newSet) {
rep.Issues = append(rep.Issues, DriftIssue{
Level: "warn",
Kind: "baseline_software",
Message: fmt.Sprintf("baseline software mismatch: baseline=%s live=%s", dctrl.Baseline.Software, newSet),
})
}
}
sort.Slice(rep.Issues, func(i, j int) bool { return rep.Issues[i].Kind < rep.Issues[j].Kind })
return rep, nil
}
+456
View File
@@ -0,0 +1,456 @@
package cluster
import (
"crypto/aes"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"golang.org/x/crypto/scrypt"
)
const (
exportMagic = "OBSCTLEXP1:"
exportVersion = 2
saltBytes = 16
scryptN = 1 << 15
scryptR = 8
scryptP = 1
)
// ExportBundle is the portable payload produced by Service.Export. It is
// self-contained: it includes every cluster (with stored credentials),
// telegram bot settings, locker config, and alert policies.
type ExportBundle struct {
Version int `json:"version"`
ExportedAt string `json:"exported_at"`
Registry Registry `json:"registry"`
Files []ExportedFile `json:"files,omitempty"`
}
type ExportedFile struct {
OriginalPath string `json:"original_path"`
Kind string `json:"kind"`
Content []byte `json:"content"`
}
// ImportMode controls how an imported bundle is merged into the current store.
type ImportMode string
const (
// ImportModeMerge adds clusters from the bundle; existing clusters with
// the same name are replaced with the imported copy.
ImportModeMerge ImportMode = "merge"
// ImportModeReplace wipes the current registry and replaces it with the
// imported bundle as-is.
ImportModeReplace ImportMode = "replace"
)
// ImportReport summarizes what happened during an import.
type ImportReport struct {
Added int
Replaced int
TotalAfter int
TelegramApplied bool
LockerApplied bool
Mode ImportMode
}
// Export writes an encrypted, passphrase-protected bundle of the full
// registry to outPath. The bundle can be imported on another machine with
// the same password — no master key transfer required.
func (s *Service) Export(outPath, password string) error {
if strings.TrimSpace(outPath) == "" {
return errors.New("export: output path is required")
}
if strings.TrimSpace(password) == "" {
return errors.New("export: password is required")
}
reg, err := s.store.Load()
if err != nil {
return fmt.Errorf("export: load registry: %w", err)
}
files, err := collectExportFiles(reg)
if err != nil {
return err
}
bundle := ExportBundle{
Version: exportVersion,
ExportedAt: s.now().UTC().Format(time.RFC3339),
Registry: reg,
Files: files,
}
payload, err := json.MarshalIndent(bundle, "", " ")
if err != nil {
return fmt.Errorf("export: encode bundle: %w", err)
}
salt := make([]byte, saltBytes)
if _, err := rand.Read(salt); err != nil {
return fmt.Errorf("export: generate salt: %w", err)
}
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
if err != nil {
return fmt.Errorf("export: derive key: %w", err)
}
ciphertext, err := encrypt(payload, key)
if err != nil {
return fmt.Errorf("export: encrypt: %w", err)
}
blob := append([]byte{}, salt...)
blob = append(blob, ciphertext...)
encoded := exportMagic + base64.StdEncoding.EncodeToString(blob) + "\n"
if dir := filepath.Dir(outPath); dir != "" && dir != "." {
if err := os.MkdirAll(dir, 0o700); err != nil {
return fmt.Errorf("export: create output dir: %w", err)
}
}
tmp := outPath + ".tmp"
if err := os.WriteFile(tmp, []byte(encoded), 0o600); err != nil {
return fmt.Errorf("export: write temp: %w", err)
}
if err := os.Rename(tmp, outPath); err != nil {
return fmt.Errorf("export: replace output: %w", err)
}
return nil
}
// Import reads an exported bundle from inPath using password, and applies
// it to the local registry according to mode.
func (s *Service) Import(inPath, password string, mode ImportMode) (ImportReport, error) {
if strings.TrimSpace(inPath) == "" {
return ImportReport{}, errors.New("import: input path is required")
}
if strings.TrimSpace(password) == "" {
return ImportReport{}, errors.New("import: password is required")
}
if mode == "" {
mode = ImportModeMerge
}
if mode != ImportModeMerge && mode != ImportModeReplace {
return ImportReport{}, fmt.Errorf("import: unsupported mode %q", mode)
}
raw, err := os.ReadFile(inPath)
if err != nil {
return ImportReport{}, fmt.Errorf("import: read input: %w", err)
}
text := strings.TrimSpace(string(raw))
if !strings.HasPrefix(text, exportMagic) {
return ImportReport{}, errors.New("import: not a PXmon export bundle")
}
blob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(text, exportMagic))
if err != nil {
return ImportReport{}, fmt.Errorf("import: decode payload: %w", err)
}
if len(blob) < saltBytes+aes.BlockSize {
return ImportReport{}, errors.New("import: payload truncated")
}
salt := blob[:saltBytes]
ciphertext := blob[saltBytes:]
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
if err != nil {
return ImportReport{}, fmt.Errorf("import: derive key: %w", err)
}
plaintext, err := decrypt(ciphertext, key)
if err != nil {
return ImportReport{}, errors.New("import: wrong password or corrupt bundle")
}
var bundle ExportBundle
if err := json.Unmarshal(plaintext, &bundle); err != nil {
return ImportReport{}, fmt.Errorf("import: parse bundle: %w", err)
}
if bundle.Version == 0 || bundle.Version > exportVersion {
return ImportReport{}, fmt.Errorf("import: unsupported bundle version %d", bundle.Version)
}
incoming := bundle.Registry
if incoming.Clusters == nil {
incoming.Clusters = []Cluster{}
}
if len(bundle.Files) > 0 {
restored, restoreErr := s.restoreExportFiles(bundle.Files)
if restoreErr != nil {
return ImportReport{}, restoreErr
}
applyRestoredFilePaths(&incoming, restored)
}
report := ImportReport{Mode: mode}
if mode == ImportModeReplace {
report.Added = len(incoming.Clusters)
report.TotalAfter = len(incoming.Clusters)
report.TelegramApplied = incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0
report.LockerApplied = incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled
if err := s.store.Save(incoming); err != nil {
return ImportReport{}, fmt.Errorf("import: save: %w", err)
}
return report, nil
}
current, err := s.store.Load()
if err != nil {
return ImportReport{}, fmt.Errorf("import: load current: %w", err)
}
if current.Clusters == nil {
current.Clusters = []Cluster{}
}
existingByName := make(map[string]int, len(current.Clusters))
for i, c := range current.Clusters {
existingByName[strings.ToLower(c.Name)] = i
}
for _, inc := range incoming.Clusters {
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key == "" {
continue
}
inc.Alerts = ensureAlertPolicy(inc.Alerts)
if idx, ok := existingByName[key]; ok {
// Preserve original ID to keep references stable.
inc.ID = current.Clusters[idx].ID
current.Clusters[idx] = inc
report.Replaced++
} else {
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
current.Clusters = append(current.Clusters, inc)
existingByName[key] = len(current.Clusters) - 1
report.Added++
}
}
if incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0 {
current.Telegram = incoming.Telegram
report.TelegramApplied = true
}
if incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled {
current.Locker = incoming.Locker
report.LockerApplied = true
}
current.Backups = mergeBackupConfig(current.Backups, incoming.Backups)
if strings.TrimSpace(current.ActiveClusterID) == "" {
current.ActiveClusterID = incoming.ActiveClusterID
}
report.TotalAfter = len(current.Clusters)
if err := s.store.Save(current); err != nil {
return ImportReport{}, fmt.Errorf("import: save: %w", err)
}
return report, nil
}
func collectExportFiles(reg Registry) ([]ExportedFile, error) {
type wantFile struct {
path string
kind string
}
wants := make([]wantFile, 0)
for _, c := range reg.Clusters {
if strings.TrimSpace(c.KeyPath) != "" {
wants = append(wants, wantFile{path: c.KeyPath, kind: "ssh_private_key"})
}
if strings.TrimSpace(c.KeyPassphraseFile) != "" {
wants = append(wants, wantFile{path: c.KeyPassphraseFile, kind: "ssh_key_passphrase"})
}
}
for _, t := range reg.Backups.Targets {
if strings.TrimSpace(t.SFTPKeyPath) != "" {
wants = append(wants, wantFile{path: t.SFTPKeyPath, kind: "sftp_private_key"})
}
}
seen := map[string]struct{}{}
files := make([]ExportedFile, 0, len(wants))
for _, w := range wants {
expanded, err := expandPath(w.path)
if err != nil {
return nil, fmt.Errorf("export: resolve %s file %q: %w", w.kind, w.path, err)
}
if _, ok := seen[expanded]; ok {
continue
}
data, err := os.ReadFile(expanded)
if err != nil {
return nil, fmt.Errorf("export: read %s file %q: %w", w.kind, expanded, err)
}
seen[expanded] = struct{}{}
files = append(files, ExportedFile{
OriginalPath: expanded,
Kind: w.kind,
Content: data,
})
}
return files, nil
}
func (s *Service) restoreExportFiles(files []ExportedFile) (map[string]string, error) {
base := filepath.Join(filepath.Dir(s.store.Path()), "imported-files")
if err := os.MkdirAll(base, 0o700); err != nil {
return nil, fmt.Errorf("import: create imported-files dir: %w", err)
}
restored := make(map[string]string, len(files))
for _, f := range files {
orig := strings.TrimSpace(f.OriginalPath)
if orig == "" {
continue
}
name := filepath.Base(orig)
if name == "." || name == string(filepath.Separator) || strings.TrimSpace(name) == "" {
name = "secret"
}
name = sanitizeExportFilename(name)
sum := sha256.Sum256([]byte(orig))
outPath := filepath.Join(base, hex.EncodeToString(sum[:6])+"-"+name)
if err := os.WriteFile(outPath, f.Content, 0o600); err != nil {
return nil, fmt.Errorf("import: restore file %q: %w", orig, err)
}
restored[orig] = outPath
}
return restored, nil
}
func applyRestoredFilePaths(reg *Registry, restored map[string]string) {
if reg == nil || len(restored) == 0 {
return
}
lookup := func(path string) string {
expanded, err := expandPath(path)
if err == nil {
if restoredPath := strings.TrimSpace(restored[expanded]); restoredPath != "" {
return restoredPath
}
}
if restoredPath := strings.TrimSpace(restored[strings.TrimSpace(path)]); restoredPath != "" {
return restoredPath
}
return path
}
for i := range reg.Clusters {
if strings.TrimSpace(reg.Clusters[i].KeyPath) != "" {
reg.Clusters[i].KeyPath = lookup(reg.Clusters[i].KeyPath)
}
if strings.TrimSpace(reg.Clusters[i].KeyPassphraseFile) != "" {
reg.Clusters[i].KeyPassphraseFile = lookup(reg.Clusters[i].KeyPassphraseFile)
}
}
for i := range reg.Backups.Targets {
if strings.TrimSpace(reg.Backups.Targets[i].SFTPKeyPath) != "" {
reg.Backups.Targets[i].SFTPKeyPath = lookup(reg.Backups.Targets[i].SFTPKeyPath)
}
}
}
func sanitizeExportFilename(name string) string {
var b strings.Builder
for _, r := range name {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= 'A' && r <= 'Z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '.', r == '_', r == '-':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
out := strings.Trim(b.String(), ".")
if out == "" {
return "secret"
}
return out
}
func mergeBackupConfig(current, incoming BackupConfig) BackupConfig {
if len(incoming.Targets) == 0 && len(incoming.Plans) == 0 {
return current
}
if current.Targets == nil {
current.Targets = []BackupTarget{}
}
if current.Plans == nil {
current.Plans = []BackupPlan{}
}
targetIDMap := map[string]string{}
targetByName := map[string]int{}
for i, t := range current.Targets {
if key := strings.ToLower(strings.TrimSpace(t.Name)); key != "" {
targetByName[key] = i
}
}
for _, inc := range incoming.Targets {
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key != "" {
if idx, ok := targetByName[key]; ok {
oldID := current.Targets[idx].ID
targetIDMap[inc.ID] = oldID
inc.ID = oldID
current.Targets[idx] = inc
continue
}
}
current.Targets = append(current.Targets, inc)
if key != "" {
targetByName[key] = len(current.Targets) - 1
}
}
planByName := map[string]int{}
for i, p := range current.Plans {
if key := strings.ToLower(strings.TrimSpace(p.Name)); key != "" {
planByName[key] = i
}
}
for _, inc := range incoming.Plans {
if mapped := strings.TrimSpace(targetIDMap[inc.TargetID]); mapped != "" {
inc.TargetID = mapped
}
if strings.TrimSpace(inc.ID) == "" {
inc.ID = newClusterID()
}
key := strings.ToLower(strings.TrimSpace(inc.Name))
if key != "" {
if idx, ok := planByName[key]; ok {
inc.ID = current.Plans[idx].ID
current.Plans[idx] = inc
continue
}
}
current.Plans = append(current.Plans, inc)
if key != "" {
planByName[key] = len(current.Plans) - 1
}
}
return current
}
+518
View File
@@ -0,0 +1,518 @@
package cluster
import (
"sort"
"strings"
"time"
)
const currentVersion = 8
// AuthMethod describes how SSH authentication is performed.
type AuthMethod string
const (
AuthMethodPassword AuthMethod = "password"
AuthMethodKey AuthMethod = "key"
)
// TransportMode selects how pxmon reaches the agent HTTP API on a node.
type TransportMode string
const (
// TransportDirect is the default: pxmon dials the agent's listen
// address over plain TCP from the local machine.
TransportDirect TransportMode = "direct"
// TransportIPFabric tunnels the agent HTTP call through the existing SSH
// connection. Meant for nodes on ipfabric-style networking where the node
// has no default outbound route and we must not touch its network config.
// The agent is expected to bind to 127.0.0.1 on the node.
TransportIPFabric TransportMode = "ipfabric"
)
func normalizeTransport(t TransportMode) TransportMode {
switch strings.ToLower(strings.TrimSpace(string(t))) {
case "ipfabric", "ip-fabric", "ip_fabric":
return TransportIPFabric
case "", "direct":
return TransportDirect
default:
return TransportDirect
}
}
// Registry is the local inventory of managed clusters (nodes).
type Registry struct {
Version int `json:"version"`
ActiveClusterID string `json:"active_cluster_id,omitempty"`
Telegram Telegram `json:"telegram,omitempty"`
Locker Locker `json:"locker,omitempty"`
Backups BackupConfig `json:"backups,omitempty"`
Clusters []Cluster `json:"clusters"`
}
type BackupConfig struct {
Targets []BackupTarget `json:"targets,omitempty"`
Plans []BackupPlan `json:"plans,omitempty"`
}
type BackupTarget struct {
ID string `json:"id"`
Name string `json:"name"`
Type string `json:"type"` // sftp|s3
Enabled bool `json:"enabled"`
SFTPHost string `json:"sftp_host,omitempty"`
SFTPPort int `json:"sftp_port,omitempty"`
SFTPUser string `json:"sftp_user,omitempty"`
SFTPPassword string `json:"sftp_password,omitempty"`
SFTPKeyPath string `json:"sftp_key_path,omitempty"`
SFTPBasePath string `json:"sftp_base_path,omitempty"`
S3Endpoint string `json:"s3_endpoint,omitempty"`
S3Region string `json:"s3_region,omitempty"`
S3Bucket string `json:"s3_bucket,omitempty"`
S3Prefix string `json:"s3_prefix,omitempty"`
S3AccessKey string `json:"s3_access_key,omitempty"`
S3SecretKey string `json:"s3_secret_key,omitempty"`
S3UseSSL bool `json:"s3_use_ssl,omitempty"`
S3PathStyle bool `json:"s3_path_style,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type BackupPlan struct {
ID string `json:"id"`
Name string `json:"name"`
Cluster string `json:"cluster,omitempty"`
TargetID string `json:"target_id"`
Paths []string `json:"paths"`
Every string `json:"every,omitempty"`
Enabled bool `json:"enabled"`
RetainDays int `json:"retain_days,omitempty"`
Compress bool `json:"compress"`
LastRunAt time.Time `json:"last_run_at,omitempty"`
LastStatus string `json:"last_status,omitempty"`
LastArchive string `json:"last_archive,omitempty"`
LastError string `json:"last_error,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// Telegram stores Telegram bot integration settings.
type Telegram struct {
Enabled bool `json:"enabled,omitempty"`
Token string `json:"token,omitempty"`
AllowedUserIDs []int64 `json:"allowed_user_ids,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
// Locker stores global UI/CLI lock settings.
type Locker struct {
Enabled bool `json:"enabled,omitempty"`
PasswordHash string `json:"password_hash,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
// Cluster describes one managed node that we can reach over SSH.
type Cluster struct {
ID string `json:"id"`
Name string `json:"name"`
Host string `json:"host"`
Port int `json:"port"`
User string `json:"user"`
Transport TransportMode `json:"transport,omitempty"`
AuthMethod AuthMethod `json:"auth_method"`
Password string `json:"password,omitempty"`
KeyPath string `json:"key_path,omitempty"`
KeyPassphrase string `json:"key_passphrase,omitempty"`
KeyPassphraseFile string `json:"key_passphrase_file,omitempty"`
InsecureHostKey bool `json:"insecure_host_key,omitempty"`
Alerts AlertPolicy `json:"alerts"`
VMAlerts VMAlertPolicy `json:"vm_alerts,omitempty"`
AlertRouting AlertRoutingPolicy `json:"alert_routing,omitempty"`
RepoTunnel RepoTunnelState `json:"repo_tunnel,omitempty"`
RunbookTrigger RunbookTrigger `json:"runbook_trigger,omitempty"`
Drift DriftControl `json:"drift,omitempty"`
Tags []string `json:"tags,omitempty"`
KVMTags map[string][]string `json:"kvm_tags,omitempty"`
Agent AgentInstall `json:"agent,omitempty"`
Software SoftwareInfo `json:"software,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// AlertPolicy defines warning thresholds used by CLI monitor mode.
type AlertPolicy struct {
CPUWarnPercent float64 `json:"cpu_warn_percent"`
RAMWarnPercent float64 `json:"ram_warn_percent"`
SwapWarnPercent float64 `json:"swap_warn_percent"`
DiskWarnPercent float64 `json:"disk_warn_percent"`
NetWarnMbps float64 `json:"net_warn_mbps"`
NetSustainEnabled bool `json:"net_sustain_enabled,omitempty"`
NetSustainIface string `json:"net_sustain_iface,omitempty"`
NetSustainInclude []string `json:"net_sustain_include,omitempty"`
NetSustainExclude []string `json:"net_sustain_exclude,omitempty"`
NetSustainMbps float64 `json:"net_sustain_mbps,omitempty"`
NetSustainMinutes int `json:"net_sustain_minutes,omitempty"`
NetSustainCooldownMins int `json:"net_sustain_cooldown_mins,omitempty"`
}
// VMAlertPolicy controls KVM VM-state alerting per cluster.
type VMAlertPolicy struct {
Enabled bool `json:"enabled,omitempty"`
WarnOnShutoff bool `json:"warn_on_shutoff,omitempty"`
MinRunning int `json:"min_running,omitempty"`
}
// AlertRoutingPolicy controls delivery behavior for alerts (e.g. Telegram).
type AlertRoutingPolicy struct {
CriticalImmediate bool `json:"critical_immediate,omitempty"`
WarningBatchMins int `json:"warning_batch_mins,omitempty"`
}
// RunbookTrigger controls automatic runbook execution on selected events.
type RunbookTrigger struct {
Enabled bool `json:"enabled,omitempty"`
OnVMShutoff bool `json:"on_vm_shutoff,omitempty"`
RunbookID string `json:"runbook_id,omitempty"`
CooldownMins int `json:"cooldown_mins,omitempty"`
LastTriggered time.Time `json:"last_triggered,omitempty"`
}
// DriftBaseline stores expected values for drift comparisons.
type DriftBaseline struct {
Enabled bool `json:"enabled,omitempty"`
SetAt time.Time `json:"set_at,omitempty"`
AgentVersion string `json:"agent_version,omitempty"`
Software string `json:"software,omitempty"`
}
// DriftControl stores baseline and per-issue acknowledgement windows.
type DriftControl struct {
Baseline DriftBaseline `json:"baseline,omitempty"`
AckUntil map[string]time.Time `json:"ack_until,omitempty"`
}
// AgentInstall describes remote pxmon-agent installation details.
type AgentInstall struct {
Installed bool `json:"installed"`
Version string `json:"version,omitempty"`
RemoteBinary string `json:"remote_binary,omitempty"`
RemoteConfig string `json:"remote_config,omitempty"`
RemoteLog string `json:"remote_log,omitempty"`
RemotePIDFile string `json:"remote_pid_file,omitempty"`
ListenAddress string `json:"listen_address,omitempty"`
Port int `json:"port,omitempty"`
Token string `json:"token,omitempty"`
RequestSecret string `json:"request_secret,omitempty"`
TLSEnabled bool `json:"tls_enabled,omitempty"`
TLSCertPath string `json:"tls_cert_path,omitempty"`
TLSKeyPath string `json:"tls_key_path,omitempty"`
TLSFingerprint string `json:"tls_fingerprint,omitempty"`
LastBootstrapAt time.Time `json:"last_bootstrap_at,omitempty"`
}
// SoftwareInfo describes discovered software/plugins on a node.
type SoftwareInfo struct {
DetectedAt time.Time `json:"detected_at,omitempty"`
Bird bool `json:"bird,omitempty"`
FRR bool `json:"frr,omitempty"`
KVM bool `json:"kvm,omitempty"`
LXC bool `json:"lxc,omitempty"`
LXD bool `json:"lxd,omitempty"`
Versions map[string]string `json:"versions,omitempty"`
}
func (s SoftwareInfo) SupportedList() []string {
out := make([]string, 0, 5)
if s.Bird {
out = append(out, "bird")
}
if s.FRR {
out = append(out, "frr")
}
if s.KVM {
out = append(out, "kvm")
}
if s.LXC {
out = append(out, "lxc")
}
if s.LXD {
out = append(out, "lxd")
}
sort.Strings(out)
return out
}
func (s SoftwareInfo) Summary() string {
list := s.SupportedList()
if len(list) == 0 {
if !s.DetectedAt.IsZero() {
return "none"
}
return "-"
}
return strings.Join(list, ",")
}
func newRegistry() Registry {
return Registry{
Version: currentVersion,
Backups: normalizeBackupConfig(BackupConfig{}),
Clusters: []Cluster{},
}
}
func defaultAlertPolicy() AlertPolicy {
return AlertPolicy{
CPUWarnPercent: 85,
RAMWarnPercent: 90,
SwapWarnPercent: 80,
DiskWarnPercent: 90,
NetWarnMbps: 300,
}
}
func defaultVMAlertPolicy() VMAlertPolicy {
return VMAlertPolicy{
Enabled: false,
WarnOnShutoff: true,
MinRunning: 1,
}
}
func defaultAlertRoutingPolicy() AlertRoutingPolicy {
return AlertRoutingPolicy{
CriticalImmediate: true,
WarningBatchMins: 5,
}
}
func defaultRunbookTrigger() RunbookTrigger {
return RunbookTrigger{
Enabled: false,
OnVMShutoff: true,
CooldownMins: 30,
}
}
func ensureAlertPolicy(p AlertPolicy) AlertPolicy {
d := defaultAlertPolicy()
if p.CPUWarnPercent <= 0 {
p.CPUWarnPercent = d.CPUWarnPercent
}
if p.RAMWarnPercent <= 0 {
p.RAMWarnPercent = d.RAMWarnPercent
}
if p.SwapWarnPercent <= 0 {
p.SwapWarnPercent = d.SwapWarnPercent
}
if p.DiskWarnPercent <= 0 {
p.DiskWarnPercent = d.DiskWarnPercent
}
if p.NetWarnMbps <= 0 {
p.NetWarnMbps = d.NetWarnMbps
}
if p.NetSustainEnabled {
if p.NetSustainMbps <= 0 {
p.NetSustainMbps = d.NetWarnMbps
}
if p.NetSustainMinutes <= 0 {
p.NetSustainMinutes = 60
}
if p.NetSustainCooldownMins <= 0 {
p.NetSustainCooldownMins = 30
}
}
return p
}
func ensureVMAlertPolicy(p VMAlertPolicy) VMAlertPolicy {
d := defaultVMAlertPolicy()
wasZero := p == (VMAlertPolicy{})
if p.MinRunning <= 0 {
p.MinRunning = d.MinRunning
}
if !p.WarnOnShutoff {
// Keep explicit false if user set it, but default to true for zero-value
// policy loaded from old configs.
if wasZero {
p.WarnOnShutoff = d.WarnOnShutoff
}
}
return p
}
func ensureAlertRoutingPolicy(p AlertRoutingPolicy) AlertRoutingPolicy {
d := defaultAlertRoutingPolicy()
if p.WarningBatchMins <= 0 {
p.WarningBatchMins = d.WarningBatchMins
}
// default true when unset
if !p.CriticalImmediate && p == (AlertRoutingPolicy{}) {
p.CriticalImmediate = d.CriticalImmediate
}
return p
}
func ensureRunbookTrigger(p RunbookTrigger) RunbookTrigger {
d := defaultRunbookTrigger()
if p.CooldownMins <= 0 {
p.CooldownMins = d.CooldownMins
}
if !p.OnVMShutoff && p == (RunbookTrigger{}) {
p.OnVMShutoff = d.OnVMShutoff
}
p.RunbookID = strings.TrimSpace(p.RunbookID)
return p
}
func normalizeBackupConfig(cfg BackupConfig) BackupConfig {
if cfg.Targets == nil {
cfg.Targets = []BackupTarget{}
}
if cfg.Plans == nil {
cfg.Plans = []BackupPlan{}
}
for i := range cfg.Targets {
t := &cfg.Targets[i]
t.ID = strings.TrimSpace(t.ID)
t.Name = strings.TrimSpace(t.Name)
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
if t.SFTPPort <= 0 {
t.SFTPPort = 22
}
if t.ID == "" {
t.ID = newClusterID()
}
if t.Name == "" {
t.Name = t.ID
}
if t.Type != "sftp" && t.Type != "s3" {
t.Type = "sftp"
}
if !t.Enabled && t.CreatedAt.IsZero() {
t.Enabled = true
}
if t.CreatedAt.IsZero() {
t.CreatedAt = time.Now().UTC()
}
if t.UpdatedAt.IsZero() {
t.UpdatedAt = t.CreatedAt
}
}
for i := range cfg.Plans {
p := &cfg.Plans[i]
p.ID = strings.TrimSpace(p.ID)
p.Name = strings.TrimSpace(p.Name)
p.Cluster = strings.TrimSpace(p.Cluster)
p.TargetID = strings.TrimSpace(p.TargetID)
if p.ID == "" {
p.ID = newClusterID()
}
if p.Name == "" {
p.Name = p.ID
}
if p.Paths == nil {
p.Paths = []string{}
}
if p.Every == "" {
p.Every = "24h"
}
if p.RetainDays <= 0 {
p.RetainDays = 30
}
if !p.Compress {
p.Compress = true
}
if !p.Enabled && p.CreatedAt.IsZero() {
p.Enabled = true
}
if p.CreatedAt.IsZero() {
p.CreatedAt = time.Now().UTC()
}
if p.UpdatedAt.IsZero() {
p.UpdatedAt = p.CreatedAt
}
}
return cfg
}
func normalizeDriftControl(d DriftControl) DriftControl {
if len(d.AckUntil) == 0 {
d.AckUntil = nil
return d
}
out := make(map[string]time.Time, len(d.AckUntil))
for k, v := range d.AckUntil {
n := strings.ToLower(strings.TrimSpace(k))
if n == "" || v.IsZero() {
continue
}
out[n] = v.UTC()
}
if len(out) == 0 {
d.AckUntil = nil
} else {
d.AckUntil = out
}
d.Baseline.AgentVersion = strings.TrimSpace(d.Baseline.AgentVersion)
d.Baseline.Software = strings.TrimSpace(d.Baseline.Software)
return d
}
func normalizeTagList(tags []string) []string {
if len(tags) == 0 {
return nil
}
seen := make(map[string]struct{}, len(tags))
out := make([]string, 0, len(tags))
for _, t := range tags {
n := strings.ToLower(strings.TrimSpace(t))
if n == "" {
continue
}
if _, ok := seen[n]; ok {
continue
}
seen[n] = struct{}{}
out = append(out, n)
}
if len(out) == 0 {
return nil
}
sort.Strings(out)
return out
}
func normalizeVMTagMap(m map[string][]string) map[string][]string {
if len(m) == 0 {
return nil
}
out := make(map[string][]string, len(m))
for vm, tags := range m {
vmName := strings.TrimSpace(vm)
if vmName == "" {
continue
}
norm := normalizeTagList(tags)
if len(norm) == 0 {
continue
}
out[vmName] = norm
}
if len(out) == 0 {
return nil
}
return out
}
func normalizeLocker(cfg Locker) Locker {
cfg.PasswordHash = strings.TrimSpace(cfg.PasswordHash)
if cfg.PasswordHash == "" {
cfg.Enabled = false
}
return cfg
}
+82
View File
@@ -0,0 +1,82 @@
package cluster
import (
"errors"
"fmt"
"math"
"strings"
"pxmon/internal/history"
)
type InterfaceP95Snapshot struct {
ClusterName string `json:"cluster"`
ClusterID string `json:"cluster_id"`
Interface string `json:"interface"`
Range history.RangeShortcut `json:"range"`
Samples int `json:"samples"`
P95Mbps float64 `json:"p95_mbps"`
AvgMbps float64 `json:"avg_mbps"`
MaxMbps float64 `json:"max_mbps"`
Series []history.NodeSamplePoint `json:"series,omitempty"`
}
func (s *Service) CollectInterfaceP95(selector, iface string, rng history.RangeShortcut) (InterfaceP95Snapshot, error) {
iface = strings.TrimSpace(iface)
if iface == "" {
return InterfaceP95Snapshot{}, errors.New("interface is required")
}
c, err := s.Get(selector)
if err != nil {
return InterfaceP95Snapshot{}, err
}
store := s.NetworkStore()
if store == nil {
return InterfaceP95Snapshot{}, errors.New("history store not configured")
}
snaps, err := store.Load(c.ID, rng.Since(s.now()))
if err != nil {
return InterfaceP95Snapshot{}, err
}
series := history.AggregateNodeSeries(snaps, iface)
out := InterfaceP95Snapshot{
ClusterName: c.Name,
ClusterID: c.ID,
Interface: iface,
Range: rng,
Samples: len(series),
P95Mbps: history.PercentileMbps(series, 95),
Series: series,
}
if len(series) == 0 {
return out, nil
}
var sum, maxV float64
for _, p := range series {
sum += p.TotalMbps
if p.TotalMbps > maxV {
maxV = p.TotalMbps
}
}
out.AvgMbps = sum / float64(len(series))
out.MaxMbps = maxV
return out, nil
}
func (s *Service) RenderInterfaceP95GraphPNG(snap InterfaceP95Snapshot) ([]byte, error) {
if len(snap.Series) == 0 {
return history.RenderNodeNetworkPNG([]history.NodeSamplePoint{}, history.ChartOptions{
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
Subtitle: "No samples",
})
}
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps", snap.P95Mbps, snap.MaxMbps, snap.AvgMbps)
if math.IsNaN(snap.P95Mbps) {
subtitle = "No samples"
}
return history.RenderNodeNetworkPNG(snap.Series, history.ChartOptions{
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
Subtitle: subtitle,
Percentile: 95,
})
}
+152
View File
@@ -0,0 +1,152 @@
package cluster
import (
"errors"
"strings"
"time"
)
func (s *Service) GetAlertRouting(selector string) (AlertRoutingPolicy, error) {
c, err := s.Get(selector)
if err != nil {
return AlertRoutingPolicy{}, err
}
return ensureAlertRoutingPolicy(c.AlertRouting), nil
}
func (s *Service) SetAlertRouting(selector string, p AlertRoutingPolicy) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.AlertRouting = ensureAlertRoutingPolicy(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("alert.routing", c.Name, "updated")
return c, nil
}
func (s *Service) GetRunbookTrigger(selector string) (RunbookTrigger, error) {
c, err := s.Get(selector)
if err != nil {
return RunbookTrigger{}, err
}
return ensureRunbookTrigger(c.RunbookTrigger), nil
}
func (s *Service) SetRunbookTrigger(selector string, p RunbookTrigger) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.RunbookTrigger = ensureRunbookTrigger(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("runbook.trigger", c.Name, "updated")
return c, nil
}
func (s *Service) TouchRunbookTrigger(selector string, when time.Time) error {
reg, err := s.store.Load()
if err != nil {
return err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return err
}
tr := ensureRunbookTrigger(c.RunbookTrigger)
tr.LastTriggered = when.UTC()
c.RunbookTrigger = tr
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
return s.store.Save(reg)
}
func (s *Service) SetDriftBaseline(selector string) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
d := normalizeDriftControl(c.Drift)
d.Baseline = DriftBaseline{
Enabled: true,
SetAt: s.now().UTC(),
AgentVersion: strings.TrimSpace(c.Agent.Version),
Software: strings.TrimSpace(c.Software.Summary()),
}
c.Drift = d
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("drift.baseline", c.Name, "set")
return c, nil
}
func (s *Service) GetDriftControl(selector string) (DriftControl, error) {
c, err := s.Get(selector)
if err != nil {
return DriftControl{}, err
}
return normalizeDriftControl(c.Drift), nil
}
func (s *Service) AckDriftIssue(selector, issueKind string, until time.Time) (Cluster, error) {
kind := strings.ToLower(strings.TrimSpace(issueKind))
if kind == "" {
return Cluster{}, errors.New("issue kind is required")
}
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
d := normalizeDriftControl(c.Drift)
if d.AckUntil == nil {
d.AckUntil = map[string]time.Time{}
}
d.AckUntil[kind] = until.UTC()
c.Drift = d
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("drift.ack", c.Name, kind+" until="+until.UTC().Format(time.RFC3339))
return c, nil
}
func (s *Service) IsDriftIssueAcked(c Cluster, kind string, now time.Time) bool {
d := normalizeDriftControl(c.Drift)
if len(d.AckUntil) == 0 {
return false
}
u, ok := d.AckUntil[strings.ToLower(strings.TrimSpace(kind))]
if !ok {
return false
}
return now.UTC().Before(u)
}
+413
View File
@@ -0,0 +1,413 @@
package cluster
import (
"context"
"errors"
"fmt"
"net"
"strconv"
"strings"
)
type RepoTunnelOptions struct {
Gateway string
GatewayIP string
Table int
Priority int
PackageManager string
Command string
KeepEnabled bool
NoRule bool
}
type RepoTunnelState struct {
Enabled bool `json:"enabled"`
Proxy string `json:"proxy,omitempty"`
Source string `json:"source,omitempty"`
}
func (s *Service) RepoTunnelEnable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
script, err := repoTunnelEnableScript(opts)
if err != nil {
return "", err
}
out, err := s.RunRemoteShell(ctx, selector, script)
if err != nil {
return "", err
}
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err == nil {
_ = s.updateRepoTunnelState(selector, RepoTunnelState{
Enabled: true,
Proxy: gw.proxyURL,
Source: strings.ToLower(strings.TrimSpace(opts.PackageManager)),
})
}
return out, nil
}
func (s *Service) RepoTunnelDisable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
script, err := repoTunnelDisableScript(opts)
if err != nil {
return "", err
}
out, err := s.RunRemoteShell(ctx, selector, script)
if err != nil {
return "", err
}
_ = s.updateRepoTunnelState(selector, RepoTunnelState{})
return out, nil
}
func (s *Service) RepoTunnelState(ctx context.Context, selector string) (RepoTunnelState, error) {
out, err := s.RunRemoteShell(ctx, selector, repoTunnelDetectScript())
if err != nil {
return RepoTunnelState{}, err
}
state := RepoTunnelState{}
for _, line := range strings.Split(out, "\n") {
line = strings.TrimSpace(line)
switch {
case strings.HasPrefix(line, "proxy="):
state.Proxy = strings.TrimSpace(strings.TrimPrefix(line, "proxy="))
case strings.HasPrefix(line, "source="):
state.Source = strings.TrimSpace(strings.TrimPrefix(line, "source="))
}
}
state.Enabled = state.Proxy != ""
return state, nil
}
func (s *Service) updateRepoTunnelState(selector string, state RepoTunnelState) error {
reg, err := s.store.Load()
if err != nil {
return err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return err
}
c.RepoTunnel = state
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
return s.store.Save(reg)
}
func (s *Service) RepoTunnelStatus(ctx context.Context, selector string) (string, error) {
return s.RunRemoteShell(ctx, selector, repoTunnelStatusScript())
}
func (s *Service) RepoTunnelInstall(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
if strings.TrimSpace(opts.Command) == "" {
return "", errors.New("install command is required")
}
enableScript, err := repoTunnelEnableScript(opts)
if err != nil {
return "", err
}
disableScript, err := repoTunnelDisableScript(opts)
if err != nil {
return "", err
}
body := enableScript + "\n" + strings.TrimSpace(opts.Command) + "\n"
if !opts.KeepEnabled {
body = enableScript + "\ncleanup_pxmon_repo_tunnel() {\n" + disableScript + "\n}\ntrap cleanup_pxmon_repo_tunnel EXIT\n" + strings.TrimSpace(opts.Command) + "\n"
}
return s.RunRemoteShell(ctx, selector, body)
}
func RepoTunnelGatewayScript(port int, allowCIDRs []string) (string, error) {
if port == 0 {
port = 3128
}
if port < 1 || port > 65535 {
return "", errors.New("--port must be in range 1..65535")
}
if len(allowCIDRs) == 0 {
return "", errors.New("at least one --allow CIDR/IP is required")
}
aclParts := make([]string, 0, len(allowCIDRs))
for _, raw := range allowCIDRs {
v := strings.TrimSpace(raw)
if v == "" {
continue
}
if !validSquidSrcACL(v) {
return "", fmt.Errorf("invalid --allow %q; use an IP or CIDR without spaces", raw)
}
aclParts = append(aclParts, v)
}
if len(aclParts) == 0 {
return "", errors.New("at least one --allow CIDR/IP is required")
}
return fmt.Sprintf(`set -eu
if command -v dnf >/dev/null 2>&1; then
dnf install -y squid
elif command -v yum >/dev/null 2>&1; then
yum install -y squid
elif command -v apt-get >/dev/null 2>&1; then
apt-get update
DEBIAN_FRONTEND=noninteractive apt-get install -y squid
else
echo "no supported package manager found for squid install" >&2
exit 1
fi
conf=/etc/squid/squid.conf
cp -a "$conf" "$conf.pxmon-bak.$(date +%%Y%%m%%d%%H%%M%%S)"
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
block=$(mktemp)
{
echo "# BEGIN PXMON REPO TUNNEL"
if ! grep -Eq "^http_port[[:space:]]+([^[:space:]]+:)?%d\b" "$conf"; then
echo "http_port %d"
fi
echo "acl pxmon_repo_tunnel src %s"
echo "http_access allow pxmon_repo_tunnel"
echo "# END PXMON REPO TUNNEL"
} > "$block"
if grep -q "^http_access deny all" "$conf"; then
awk -v block="$block" '
BEGIN {while ((getline line < block) > 0) b = b line "\n"; close(block); inserted=0}
/^http_access deny all/ && !inserted {printf "%%s", b; inserted=1}
{print}
END {if (!inserted) printf "%%s", b}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
else
cat "$block" >> "$conf"
fi
rm -f "$block"
systemctl enable --now squid
systemctl restart squid
if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
firewall-cmd --add-port=%d/tcp --permanent
firewall-cmd --reload
fi
echo "pxmon repo gateway ready on port %d"
`, port, port, strings.Join(aclParts, " "), port, port), nil
}
func validSquidSrcACL(v string) bool {
for _, r := range v {
if r >= 'a' && r <= 'z' {
continue
}
if r >= 'A' && r <= 'Z' {
continue
}
if r >= '0' && r <= '9' {
continue
}
switch r {
case '.', ':', '/', '_', '-':
continue
default:
return false
}
}
return v != ""
}
func repoTunnelEnableScript(opts RepoTunnelOptions) (string, error) {
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err != nil {
return "", err
}
if !opts.NoRule && opts.Table <= 0 {
return "", errors.New("--table is required unless --no-rule is used")
}
manager := strings.ToLower(strings.TrimSpace(opts.PackageManager))
if manager == "" {
manager = "auto"
}
if manager != "auto" && manager != "apt" && manager != "dnf" && manager != "yum" {
return "", fmt.Errorf("unsupported --manager %q", opts.PackageManager)
}
ruleLine := ""
if !opts.NoRule {
addCmd := fmt.Sprintf("ip rule add to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Table)
if opts.Priority > 0 {
addCmd = fmt.Sprintf("ip rule add priority %d to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Priority, opts.Table)
}
ruleLine = fmt.Sprintf(`
if ! ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; then
%s
fi`, opts.Table, addCmd)
}
return fmt.Sprintf(`set -eu
PXMON_GATEWAY_HOST=%s
PXMON_GATEWAY_IP=%s
PXMON_PROXY_URL=%s
PXMON_MANAGER=%s
if [ -z "$PXMON_GATEWAY_IP" ]; then
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
fi
if [ -z "$PXMON_GATEWAY_IP" ]; then
echo "cannot resolve repo gateway: $PXMON_GATEWAY_HOST" >&2
exit 1
fi
%s
pxmon_repo_manager="$PXMON_MANAGER"
if [ "$pxmon_repo_manager" = "auto" ]; then
if command -v apt-get >/dev/null 2>&1; then pxmon_repo_manager=apt
elif command -v dnf >/dev/null 2>&1; then pxmon_repo_manager=dnf
elif command -v yum >/dev/null 2>&1; then pxmon_repo_manager=yum
else echo "no supported package manager found" >&2; exit 1
fi
fi
case "$pxmon_repo_manager" in
apt)
mkdir -p /etc/apt/apt.conf.d
cat > /etc/apt/apt.conf.d/99-pxmon-repo-tunnel <<EOF
Acquire::http::Proxy "$PXMON_PROXY_URL";
Acquire::https::Proxy "$PXMON_PROXY_URL";
EOF
;;
dnf|yum)
conf=/etc/dnf/dnf.conf
[ "$pxmon_repo_manager" = "yum" ] && conf=/etc/yum.conf
[ -f "$conf" ] || touch "$conf"
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
{
echo "# BEGIN PXMON REPO TUNNEL"
echo "proxy=$PXMON_PROXY_URL"
echo "# END PXMON REPO TUNNEL"
} >> "$conf"
;;
esac
echo "pxmon repo tunnel enabled: proxy=$PXMON_PROXY_URL gateway_ip=$PXMON_GATEWAY_IP manager=$pxmon_repo_manager"
`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), shellQuote(gw.proxyURL), shellQuote(manager), ruleLine), nil
}
func repoTunnelDisableScript(opts RepoTunnelOptions) (string, error) {
gw, err := parseRepoTunnelGateway(opts.Gateway)
if err != nil && !opts.NoRule {
return "", err
}
ruleLine := ""
if !opts.NoRule {
if opts.Table <= 0 {
return "", errors.New("--table is required unless --no-rule is used")
}
ruleLine = fmt.Sprintf(`
PXMON_GATEWAY_HOST=%s
PXMON_GATEWAY_IP=%s
if [ -z "$PXMON_GATEWAY_IP" ]; then
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
fi
if [ -n "$PXMON_GATEWAY_IP" ]; then
while ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; do
ip rule del to "$PXMON_GATEWAY_IP/32" table %d 2>/dev/null || break
done
fi`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), opts.Table, opts.Table)
}
return fmt.Sprintf(`set -eu
rm -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
if [ -f "$conf" ]; then
awk '
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
/# END PXMON REPO TUNNEL/ {skip=0; next}
!skip {print}
' "$conf" > "$conf.tmp"
mv "$conf.tmp" "$conf"
fi
done
%s
echo "pxmon repo tunnel disabled"
`, ruleLine), nil
}
func repoTunnelStatusScript() string {
return `set -eu
echo "== ip rules =="
ip rule show | grep -E "lookup|table" || true
echo
echo "== apt proxy =="
[ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ] && cat /etc/apt/apt.conf.d/99-pxmon-repo-tunnel || echo "(none)"
echo
echo "== dnf/yum proxy =="
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
[ -f "$conf" ] || continue
echo "-- $conf"
awk '/# BEGIN PXMON REPO TUNNEL/,/# END PXMON REPO TUNNEL/ {print}' "$conf"
done`
}
func repoTunnelDetectScript() string {
return `set -eu
if [ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ]; then
proxy=$(sed -n 's/.*Proxy[[:space:]]*"\([^"]*\)".*/\1/p' /etc/apt/apt.conf.d/99-pxmon-repo-tunnel | head -1)
[ -n "$proxy" ] && printf 'proxy=%s\nsource=apt\n' "$proxy" && exit 0
fi
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
[ -f "$conf" ] || continue
proxy=$(awk '
/# BEGIN PXMON REPO TUNNEL/ {inside=1; next}
/# END PXMON REPO TUNNEL/ {inside=0; next}
inside && /^proxy[[:space:]]*=/ {
sub(/^[^=]*=/, "")
gsub(/^[[:space:]]+|[[:space:]]+$/, "")
print
exit
}
' "$conf")
[ -n "$proxy" ] && printf 'proxy=%s\nsource=%s\n' "$proxy" "$conf" && exit 0
done
exit 0`
}
type repoTunnelGateway struct {
host string
port int
proxyURL string
}
func parseRepoTunnelGateway(raw string) (repoTunnelGateway, error) {
v := strings.TrimSpace(raw)
if v == "" {
return repoTunnelGateway{}, errors.New("--gateway is required")
}
if strings.HasPrefix(v, "http://") {
v = strings.TrimPrefix(v, "http://")
}
if strings.HasPrefix(v, "https://") {
return repoTunnelGateway{}, errors.New("--gateway must be an http proxy endpoint, not https")
}
host, portRaw, err := net.SplitHostPort(v)
if err != nil {
if strings.Count(v, ":") > 1 {
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway %q; use host:port or [ipv6]:port", raw)
}
host = v
portRaw = "3128"
}
host = strings.Trim(host, "[]")
if strings.TrimSpace(host) == "" {
return repoTunnelGateway{}, errors.New("--gateway host is empty")
}
port, err := strconv.Atoi(portRaw)
if err != nil || port < 1 || port > 65535 {
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway port %q", portRaw)
}
return repoTunnelGateway{
host: host,
port: port,
proxyURL: "http://" + net.JoinHostPort(host, strconv.Itoa(port)),
}, nil
}
+187
View File
@@ -0,0 +1,187 @@
package cluster
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
type RunbookStep struct {
Title string `json:"title"`
Command string `json:"command,omitempty"`
Note string `json:"note,omitempty"`
}
type Runbook struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Steps []RunbookStep `json:"steps"`
BuiltIn bool `json:"built_in,omitempty"`
CreatedAt time.Time `json:"created_at,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}
func builtinRunbooks() []Runbook {
return []Runbook{
{
ID: "vm-health-check",
Name: "VM Health Check",
Description: "Quick validation of agent, VM inventory and VM alert policy.",
BuiltIn: true,
Steps: []RunbookStep{
{Title: "Check agent status", Command: "cluster agent status"},
{Title: "Check cluster drift", Command: "cluster drift"},
{Title: "Check VM states", Command: "cluster alert-vm check"},
{Title: "Inspect VM allocations", Command: "kvm top"},
},
},
{
ID: "traffic-billing-audit",
Name: "Traffic Billing Audit",
Description: "Collect interface P95 and graph evidence for billing period.",
BuiltIn: true,
Steps: []RunbookStep{
{Title: "List interfaces", Command: "cluster usage --range 1h"},
{Title: "Compute P95 for target iface", Command: "cluster p95 --iface <iface> --range 30d --graph"},
{Title: "Export report", Command: "cluster report export --format json --out ./report.json"},
},
},
}
}
func (s *Service) runbookPath() string {
return filepath.Join(s.DataDir(), "runbooks", "custom.json")
}
func (s *Service) loadCustomRunbooks() ([]Runbook, error) {
path := s.runbookPath()
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return []Runbook{}, nil
}
return nil, err
}
if len(strings.TrimSpace(string(raw))) == 0 {
return []Runbook{}, nil
}
var items []Runbook
if err := json.Unmarshal(raw, &items); err != nil {
return nil, err
}
out := make([]Runbook, 0, len(items))
for _, rb := range items {
rb.ID = strings.TrimSpace(rb.ID)
rb.Name = strings.TrimSpace(rb.Name)
if rb.ID == "" || rb.Name == "" {
continue
}
rb.BuiltIn = false
out = append(out, rb)
}
return out, nil
}
func (s *Service) saveCustomRunbooks(items []Runbook) error {
path := s.runbookPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
b, err := json.MarshalIndent(items, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
tmp := path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *Service) ListRunbooks() ([]Runbook, error) {
built := builtinRunbooks()
custom, err := s.loadCustomRunbooks()
if err != nil {
return nil, err
}
all := append(append([]Runbook{}, built...), custom...)
sort.Slice(all, func(i, j int) bool { return strings.ToLower(all[i].ID) < strings.ToLower(all[j].ID) })
return all, nil
}
func (s *Service) GetRunbook(selector string) (Runbook, bool) {
selector = strings.TrimSpace(selector)
items, err := s.ListRunbooks()
if err != nil {
return Runbook{}, false
}
for _, rb := range items {
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
return rb, true
}
}
return Runbook{}, false
}
func (s *Service) AddRunbook(rb Runbook) (Runbook, error) {
rb.ID = strings.TrimSpace(rb.ID)
rb.Name = strings.TrimSpace(rb.Name)
if rb.ID == "" || rb.Name == "" {
return Runbook{}, errors.New("runbook id and name are required")
}
if len(rb.Steps) == 0 {
return Runbook{}, errors.New("runbook steps are required")
}
if b, ok := s.GetRunbook(rb.ID); ok && b.BuiltIn {
return Runbook{}, errors.New("cannot overwrite built-in runbook")
}
custom, err := s.loadCustomRunbooks()
if err != nil {
return Runbook{}, err
}
for _, it := range custom {
if strings.EqualFold(it.ID, rb.ID) {
return Runbook{}, errors.New("runbook id already exists")
}
}
now := s.now().UTC()
rb.BuiltIn = false
rb.CreatedAt = now
rb.UpdatedAt = now
custom = append(custom, rb)
if err := s.saveCustomRunbooks(custom); err != nil {
return Runbook{}, err
}
_ = s.AppendChange("runbook.add", rb.ID, rb.Name)
return rb, nil
}
func (s *Service) RemoveRunbook(selector string) (Runbook, error) {
custom, err := s.loadCustomRunbooks()
if err != nil {
return Runbook{}, err
}
idx := -1
for i, rb := range custom {
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return Runbook{}, errors.New("custom runbook not found")
}
removed := custom[idx]
custom = append(custom[:idx], custom[idx+1:]...)
if err := s.saveCustomRunbooks(custom); err != nil {
return Runbook{}, err
}
_ = s.AppendChange("runbook.remove", removed.ID, removed.Name)
return removed, nil
}
+301
View File
@@ -0,0 +1,301 @@
package cluster
import (
"encoding/json"
"errors"
"fmt"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
type ScheduledTask struct {
ID string `json:"id"`
Name string `json:"name"`
Cluster string `json:"cluster,omitempty"`
Command string `json:"command"`
Mode string `json:"mode,omitempty"` // shell|observer
Every string `json:"every"`
Backoff string `json:"backoff,omitempty"`
JitterSec int `json:"jitter_sec,omitempty"`
RetryMax int `json:"retry_max,omitempty"`
RetryCur int `json:"retry_cur,omitempty"`
Enabled bool `json:"enabled"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
LastRunAt time.Time `json:"last_run_at,omitempty"`
NextRunAt time.Time `json:"next_run_at,omitempty"`
}
type SchedulerRunResult struct {
Task ScheduledTask `json:"task"`
Ran bool `json:"ran"`
Error string `json:"error,omitempty"`
Output string `json:"output,omitempty"`
ExitCode int `json:"exit_code,omitempty"`
}
func (s *Service) schedulerPath() string {
return filepath.Join(s.DataDir(), "scheduler", "tasks.json")
}
func (s *Service) loadTasks() ([]ScheduledTask, error) {
path := s.schedulerPath()
raw, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
return []ScheduledTask{}, nil
}
return nil, err
}
var items []ScheduledTask
if len(strings.TrimSpace(string(raw))) == 0 {
return []ScheduledTask{}, nil
}
if err := json.Unmarshal(raw, &items); err != nil {
return nil, err
}
for i := range items {
items[i] = normalizeScheduledTask(items[i])
}
sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
return items, nil
}
func (s *Service) saveTasks(items []ScheduledTask) error {
path := s.schedulerPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
b, err := json.MarshalIndent(items, "", " ")
if err != nil {
return err
}
b = append(b, '\n')
tmp := path + ".tmp"
if err := os.WriteFile(tmp, b, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func normalizeScheduledTask(t ScheduledTask) ScheduledTask {
t.ID = strings.TrimSpace(t.ID)
t.Name = strings.TrimSpace(t.Name)
t.Command = strings.TrimSpace(t.Command)
t.Cluster = strings.TrimSpace(t.Cluster)
t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
t.Every = strings.TrimSpace(t.Every)
if t.ID == "" {
t.ID = newClusterID()
}
if t.Name == "" {
t.Name = t.ID
}
if t.Every == "" {
t.Every = "5m"
}
if strings.TrimSpace(t.Backoff) == "" {
t.Backoff = "30s"
}
if t.JitterSec < 0 {
t.JitterSec = 0
}
if t.RetryMax <= 0 {
t.RetryMax = 3
}
if t.RetryCur < 0 {
t.RetryCur = 0
}
if t.Mode == "" {
t.Mode = "shell"
}
if t.Mode != "shell" && t.Mode != "observer" {
t.Mode = "shell"
}
if t.CreatedAt.IsZero() {
t.CreatedAt = time.Now().UTC()
}
if t.UpdatedAt.IsZero() {
t.UpdatedAt = t.CreatedAt
}
return t
}
func parseTaskEvery(v string) (time.Duration, error) {
d, err := time.ParseDuration(strings.TrimSpace(v))
if err != nil {
return 0, fmt.Errorf("invalid --every duration: %w", err)
}
if d < time.Minute {
return 0, errors.New("--every must be >= 1m")
}
return d, nil
}
func (s *Service) SchedulerList() ([]ScheduledTask, error) {
return s.loadTasks()
}
func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) {
name = strings.TrimSpace(name)
command = strings.TrimSpace(command)
every = strings.TrimSpace(every)
mode = strings.ToLower(strings.TrimSpace(mode))
if name == "" {
return ScheduledTask{}, errors.New("task name is required")
}
if command == "" {
return ScheduledTask{}, errors.New("task command is required")
}
if mode == "" {
mode = "shell"
}
if mode != "shell" && mode != "observer" {
return ScheduledTask{}, errors.New("task mode must be shell|observer")
}
d, err := parseTaskEvery(every)
if err != nil {
return ScheduledTask{}, err
}
if strings.TrimSpace(backoff) == "" {
backoff = "30s"
}
if _, err := time.ParseDuration(backoff); err != nil {
return ScheduledTask{}, errors.New("invalid backoff duration")
}
if jitterSec < 0 {
jitterSec = 0
}
if retryMax <= 0 {
retryMax = 3
}
items, err := s.loadTasks()
if err != nil {
return ScheduledTask{}, err
}
for _, t := range items {
if strings.EqualFold(t.Name, name) {
return ScheduledTask{}, fmt.Errorf("task %q already exists", name)
}
}
now := s.now().UTC()
t := ScheduledTask{
ID: newClusterID(),
Name: name,
Cluster: strings.TrimSpace(clusterSel),
Command: command,
Mode: mode,
Every: every,
Backoff: backoff,
JitterSec: jitterSec,
RetryMax: retryMax,
Enabled: enabled,
CreatedAt: now,
UpdatedAt: now,
}
if enabled {
t.NextRunAt = now.Add(d)
}
items = append(items, t)
if err := s.saveTasks(items); err != nil {
return ScheduledTask{}, err
}
_ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every))
return t, nil
}
func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) {
items, err := s.loadTasks()
if err != nil {
return ScheduledTask{}, err
}
selector = strings.TrimSpace(selector)
if selector == "" {
return ScheduledTask{}, errors.New("task name or id is required")
}
idx := -1
for i, t := range items {
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return ScheduledTask{}, errors.New("task not found")
}
removed := items[idx]
items = append(items[:idx], items[idx+1:]...)
if err := s.saveTasks(items); err != nil {
return ScheduledTask{}, err
}
_ = s.AppendChange("scheduler.remove", removed.Name, removed.Command)
return removed, nil
}
func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error {
items, err := s.loadTasks()
if err != nil {
return err
}
for i := range items {
if items[i].ID != taskID {
continue
}
items[i].LastRunAt = ranAt.UTC()
if success {
d, err := parseTaskEvery(items[i].Every)
if err != nil {
return err
}
items[i].RetryCur = 0
items[i].NextRunAt = items[i].LastRunAt.Add(d)
} else {
items[i].RetryCur++
if items[i].RetryCur > items[i].RetryMax {
// cap retries and move to the next normal run window
items[i].RetryCur = 0
d, err := parseTaskEvery(items[i].Every)
if err != nil {
return err
}
items[i].NextRunAt = items[i].LastRunAt.Add(d)
} else {
back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff))
if err != nil || back <= 0 {
back = 30 * time.Second
}
delay := back * time.Duration(1<<(items[i].RetryCur-1))
if items[i].JitterSec > 0 {
delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second
}
items[i].NextRunAt = items[i].LastRunAt.Add(delay)
}
}
items[i].UpdatedAt = ranAt.UTC()
break
}
return s.saveTasks(items)
}
func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) {
items, err := s.loadTasks()
if err != nil {
return nil, err
}
n := now.UTC()
due := make([]ScheduledTask, 0)
for _, t := range items {
if !t.Enabled {
continue
}
if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) {
due = append(due, t)
}
}
sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) })
return due, nil
}
File diff suppressed because it is too large Load Diff
+554
View File
@@ -0,0 +1,554 @@
package cluster
import (
"context"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
func newTestService(t *testing.T) *Service {
t.Helper()
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
return NewService(store)
}
func TestConnectListUseDisconnect(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
c1, _, err := svc.Connect(ctx, ConnectOptions{
Name: "eu-1",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass1",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect c1 error: %v", err)
}
c2, _, err := svc.Connect(ctx, ConnectOptions{
Name: "us-1",
Host: "10.0.0.11",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect c2 error: %v", err)
}
clusters, activeID, err := svc.List()
if err != nil {
t.Fatalf("list error: %v", err)
}
if len(clusters) != 2 {
t.Fatalf("expected 2 clusters, got %d", len(clusters))
}
if activeID != c2.ID {
t.Fatalf("expected active %s, got %s", c2.ID, activeID)
}
_, err = svc.Use(c1.Name)
if err != nil {
t.Fatalf("use error: %v", err)
}
current, err := svc.Current()
if err != nil {
t.Fatalf("current error: %v", err)
}
if current.ID != c1.ID {
t.Fatalf("expected current %s, got %s", c1.ID, current.ID)
}
_, err = svc.Disconnect(c1.Name)
if err != nil {
t.Fatalf("disconnect error: %v", err)
}
current, err = svc.Current()
if err != nil {
t.Fatalf("current after disconnect error: %v", err)
}
if current.ID != c2.ID {
t.Fatalf("expected fallback current %s, got %s", c2.ID, current.ID)
}
}
func TestConnectDuplicateNameRequiresForce(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass1",
SkipCheck: true,
})
if err != nil {
t.Fatalf("initial connect error: %v", err)
}
_, _, err = svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.20",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
})
if err == nil {
t.Fatal("expected duplicate name error")
}
c, _, err := svc.Connect(ctx, ConnectOptions{
Name: "prod",
Host: "10.0.0.20",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass2",
SkipCheck: true,
Force: true,
})
if err != nil {
t.Fatalf("force connect error: %v", err)
}
if c.Host != "10.0.0.20" {
t.Fatalf("expected overwritten host, got %s", c.Host)
}
}
func TestExportImportRestoresKeyFilesAndNewFields(t *testing.T) {
t.Parallel()
srcDir := t.TempDir()
keyPath := filepath.Join(srcDir, "id_ed25519")
passPath := filepath.Join(srcDir, "pass.pxmonpassphrase")
sftpKeyPath := filepath.Join(srcDir, "sftp_key")
if err := os.WriteFile(keyPath, []byte("PRIVATE KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(passPath, []byte("secret-pass\n"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(sftpKeyPath, []byte("SFTP PRIVATE KEY\n"), 0o600); err != nil {
t.Fatal(err)
}
src := newTestService(t)
reg := newRegistry()
reg.ActiveClusterID = "clu_1"
reg.Backups.Targets = []BackupTarget{{
ID: "bt_1",
Name: "sftp",
Type: "sftp",
Enabled: true,
SFTPKeyPath: sftpKeyPath,
}}
reg.Clusters = []Cluster{{
ID: "clu_1",
Name: "node",
Host: "192.0.2.10",
Port: 22,
User: "root",
Transport: TransportIPFabric,
AuthMethod: AuthMethodKey,
KeyPath: keyPath,
KeyPassphraseFile: passPath,
RepoTunnel: RepoTunnelState{
Enabled: true,
Proxy: "http://203.0.113.10:3128",
Source: "dnf",
},
Alerts: defaultAlertPolicy(),
CreatedAt: time.Now().UTC(),
UpdatedAt: time.Now().UTC(),
}}
if err := src.store.Save(reg); err != nil {
t.Fatalf("save source registry: %v", err)
}
bundlePath := filepath.Join(t.TempDir(), "pxmon-export.enc")
if err := src.Export(bundlePath, "strong-test-passphrase"); err != nil {
t.Fatalf("export error: %v", err)
}
raw, err := os.ReadFile(bundlePath)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(raw), "PRIVATE KEY") || strings.Contains(string(raw), "secret-pass") {
t.Fatal("export bundle leaked key material in plaintext")
}
dst := newTestService(t)
if _, err := dst.Import(bundlePath, "strong-test-passphrase", ImportModeReplace); err != nil {
t.Fatalf("import error: %v", err)
}
gotReg, err := dst.store.Load()
if err != nil {
t.Fatalf("load imported registry: %v", err)
}
if len(gotReg.Clusters) != 1 {
t.Fatalf("expected 1 cluster, got %d", len(gotReg.Clusters))
}
got := gotReg.Clusters[0]
if !got.RepoTunnel.Enabled || got.RepoTunnel.Proxy != "http://203.0.113.10:3128" {
t.Fatalf("repo tunnel was not preserved: %+v", got.RepoTunnel)
}
if got.KeyPath == keyPath || got.KeyPassphraseFile == passPath {
t.Fatalf("expected key paths to be restored under destination config dir, got key=%q pass=%q", got.KeyPath, got.KeyPassphraseFile)
}
keyData, err := os.ReadFile(got.KeyPath)
if err != nil {
t.Fatalf("read restored key: %v", err)
}
if string(keyData) != "PRIVATE KEY\n" {
t.Fatalf("unexpected restored key content: %q", string(keyData))
}
passData, err := os.ReadFile(got.KeyPassphraseFile)
if err != nil {
t.Fatalf("read restored passphrase file: %v", err)
}
if string(passData) != "secret-pass\n" {
t.Fatalf("unexpected restored passphrase content: %q", string(passData))
}
if len(gotReg.Backups.Targets) != 1 || gotReg.Backups.Targets[0].SFTPKeyPath == sftpKeyPath {
t.Fatalf("expected restored SFTP key path, got %+v", gotReg.Backups.Targets)
}
mergeDst := newTestService(t)
if _, err := mergeDst.Import(bundlePath, "strong-test-passphrase", ImportModeMerge); err != nil {
t.Fatalf("merge import error: %v", err)
}
mergeReg, err := mergeDst.store.Load()
if err != nil {
t.Fatalf("load merge registry: %v", err)
}
if len(mergeReg.Backups.Targets) != 1 || strings.TrimSpace(mergeReg.Backups.Targets[0].SFTPKeyPath) == "" {
t.Fatalf("expected backup target to be merged, got %+v", mergeReg.Backups)
}
}
func TestConnectRequiresAuthData(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "bad",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
SkipCheck: true,
})
if err == nil {
t.Fatal("expected error for empty password auth")
}
}
func TestAlertPolicySetAndGet(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
_, _, err := svc.Connect(ctx, ConnectOptions{
Name: "node-1",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect error: %v", err)
}
updated, err := svc.SetAlertPolicy("node-1", AlertPolicy{
CPUWarnPercent: 70,
RAMWarnPercent: 75,
SwapWarnPercent: 60,
DiskWarnPercent: 80,
NetWarnMbps: 120,
NetSustainEnabled: true,
NetSustainIface: "eth0",
NetSustainInclude: []string{"net0"},
NetSustainExclude: []string{"backup"},
NetSustainMbps: 500,
NetSustainMinutes: 60,
NetSustainCooldownMins: 15,
})
if err != nil {
t.Fatalf("set alert policy error: %v", err)
}
if updated.Alerts.NetWarnMbps != 120 {
t.Fatalf("unexpected net threshold: %.2f", updated.Alerts.NetWarnMbps)
}
got, err := svc.GetAlertPolicy("node-1")
if err != nil {
t.Fatalf("get alert policy error: %v", err)
}
if got.RAMWarnPercent != 75 || got.DiskWarnPercent != 80 {
t.Fatalf("unexpected thresholds: %+v", got)
}
if !got.NetSustainEnabled || got.NetSustainIface != "eth0" || got.NetSustainMbps != 500 {
t.Fatalf("unexpected sustained net policy: %+v", got)
}
if len(got.NetSustainInclude) != 1 || got.NetSustainInclude[0] != "net0" {
t.Fatalf("unexpected sustained net include filter: %+v", got.NetSustainInclude)
}
if len(got.NetSustainExclude) != 1 || got.NetSustainExclude[0] != "backup" {
t.Fatalf("unexpected sustained net exclude filter: %+v", got.NetSustainExclude)
}
}
func TestIsPluginToolSupported(t *testing.T) {
t.Parallel()
info := SoftwareInfo{
Bird: true,
FRR: true,
KVM: true,
LXC: true,
LXD: false,
}
tests := []struct {
tool string
ok bool
}{
{tool: "bird", ok: true},
{tool: "frr", ok: true},
{tool: "kvm", ok: true},
{tool: "lxc", ok: true},
{tool: "lxd", ok: true}, // lxd aliases to lxc command templates
{tool: "unknown", ok: false},
}
for _, tt := range tests {
tt := tt
t.Run(tt.tool, func(t *testing.T) {
t.Parallel()
if got := isPluginToolSupported(info, tt.tool); got != tt.ok {
t.Fatalf("tool=%s expected %v got %v", tt.tool, tt.ok, got)
}
})
}
}
func TestLXDTopUsesTemplateNotRawLxcTop(t *testing.T) {
t.Parallel()
script, err := pluginScript("lxd", "top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
if strings.Contains(script, "lxc top") {
t.Fatalf("expected custom template script, got raw lxc top: %q", script)
}
if !strings.Contains(script, "lxc info") {
t.Fatalf("expected lxc info usage in top template")
}
}
func TestRunPluginActionReportsMissingSupport(t *testing.T) {
t.Parallel()
svc := newTestService(t)
ctx := context.Background()
cluster, _, err := svc.Connect(ctx, ConnectOptions{
Name: "eu-1",
Host: "127.0.0.1",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "pass",
SkipCheck: true,
})
if err != nil {
t.Fatalf("connect error: %v", err)
}
reg, err := svc.store.Load()
if err != nil {
t.Fatalf("load registry error: %v", err)
}
for i := range reg.Clusters {
if reg.Clusters[i].ID == cluster.ID {
reg.Clusters[i].Software = SoftwareInfo{
DetectedAt: time.Now().UTC(),
}
break
}
}
if err := svc.store.Save(reg); err != nil {
t.Fatalf("save registry error: %v", err)
}
_, err = svc.RunPluginAction(ctx, cluster.Name, "lxd", "top", nil)
if err == nil {
t.Fatalf("expected unsupported software error")
}
if !strings.Contains(err.Error(), "support for lxd was not detected") {
t.Fatalf("unexpected error message: %v", err)
}
}
func TestKVMTopScriptIncludesReadableMetrics(t *testing.T) {
t.Parallel()
script, err := pluginScript("kvm", "top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
for _, want := range []string{
"VCPU",
"RAM_MAX",
"DISK_CAP",
"DISK_ALLOC",
} {
if !strings.Contains(script, want) {
t.Fatalf("expected %q in kvm top script", want)
}
}
}
func TestKVMNetTopScriptIncludesRateAndP95(t *testing.T) {
t.Parallel()
script, err := pluginScript("kvm", "net-top", nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
for _, want := range []string{
"NET_Mbps",
"RX_Mbps",
"TX_Mbps",
"P95_Mbps",
"RX_TOTAL",
"TX_TOTAL",
} {
if !strings.Contains(script, want) {
t.Fatalf("expected %q in kvm net-top script", want)
}
}
}
func TestKVMScriptsAreShellParseable(t *testing.T) {
t.Parallel()
cases := []struct {
tool string
action string
}{
{tool: "kvm", action: "top"},
{tool: "kvm", action: "net-top"},
}
for _, tc := range cases {
tc := tc
t.Run(tc.tool+"-"+tc.action, func(t *testing.T) {
t.Parallel()
script, err := pluginScript(tc.tool, tc.action, nil)
if err != nil {
t.Fatalf("pluginScript error: %v", err)
}
cmd := exec.Command("sh", "-n", "-c", script)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("shell parse failed: %v\n%s\nSCRIPT:\n%s", err, string(out), script)
}
})
}
}
func TestTelegramConfigSetGetDisable(t *testing.T) {
t.Parallel()
svc := newTestService(t)
cfg, err := svc.SetTelegram(Telegram{
Enabled: true,
Token: "123:ABC",
AllowedUserIDs: []int64{2002, 1001, 2002},
})
if err != nil {
t.Fatalf("set telegram config: %v", err)
}
if !cfg.Enabled {
t.Fatal("expected enabled telegram config")
}
if len(cfg.AllowedUserIDs) != 2 {
t.Fatalf("expected deduped ids, got %+v", cfg.AllowedUserIDs)
}
got, err := svc.GetTelegram()
if err != nil {
t.Fatalf("get telegram config: %v", err)
}
if got.Token != "123:ABC" {
t.Fatalf("unexpected token: %q", got.Token)
}
if len(got.AllowedUserIDs) != 2 || got.AllowedUserIDs[0] != 1001 || got.AllowedUserIDs[1] != 2002 {
t.Fatalf("unexpected allowed ids: %+v", got.AllowedUserIDs)
}
disabled, err := svc.DisableTelegram()
if err != nil {
t.Fatalf("disable telegram config: %v", err)
}
if disabled.Enabled {
t.Fatal("expected disabled telegram config")
}
}
func TestTelegramConfigValidation(t *testing.T) {
t.Parallel()
svc := newTestService(t)
_, err := svc.SetTelegram(Telegram{
Enabled: true,
AllowedUserIDs: []int64{123},
})
if err == nil {
t.Fatal("expected token validation error")
}
_, err = svc.SetTelegram(Telegram{
Enabled: true,
Token: "123:ABC",
})
if err == nil {
t.Fatal("expected allowed ids validation error")
}
}
+159
View File
@@ -0,0 +1,159 @@
package cluster
import (
"math"
"sort"
"strings"
"time"
"pxmon/internal/history"
)
type AvailabilityVM struct {
Name string `json:"name"`
Availability float64 `json:"availability_pct"`
Samples int `json:"samples"`
Running int `json:"running_samples"`
}
type AvailabilityReport struct {
Cluster string `json:"cluster"`
Range string `json:"range"`
Samples int `json:"samples"`
UpSamples int `json:"up_samples"`
Availability float64 `json:"availability_pct"`
VMs []AvailabilityVM `json:"vms,omitempty"`
}
func (s *Service) AvailabilityReport(selector string, since time.Time, vmFilter string) (AvailabilityReport, error) {
c, err := s.Get(selector)
if err != nil {
return AvailabilityReport{}, err
}
store := history.NewAvailabilityStore(s.DataDir())
snaps, err := store.Load(c.ID, since)
if err != nil {
return AvailabilityReport{}, err
}
rep := AvailabilityReport{Cluster: c.Name, Samples: len(snaps)}
if len(snaps) == 0 {
return rep, nil
}
vmFilter = strings.TrimSpace(vmFilter)
totalUp := 0
type acc struct{ samples, running int }
vmap := map[string]*acc{}
for _, snap := range snaps {
if snap.ClusterUp {
totalUp++
}
for vm, st := range snap.VMStates {
if vmFilter != "" && !strings.EqualFold(vmFilter, vm) {
continue
}
a := vmap[vm]
if a == nil {
a = &acc{}
vmap[vm] = a
}
a.samples++
if strings.EqualFold(strings.TrimSpace(st), "running") {
a.running++
}
}
}
rep.UpSamples = totalUp
rep.Availability = 100 * float64(totalUp) / float64(len(snaps))
for vm, a := range vmap {
if a.samples == 0 {
continue
}
rep.VMs = append(rep.VMs, AvailabilityVM{
Name: vm,
Samples: a.samples,
Running: a.running,
Availability: 100 * float64(a.running) / float64(a.samples),
})
}
sort.Slice(rep.VMs, func(i, j int) bool { return rep.VMs[i].Name < rep.VMs[j].Name })
return rep, nil
}
type CapacityForecastItem struct {
Mount string `json:"mount"`
UsedPct float64 `json:"used_pct"`
SlopeBytesSec float64 `json:"slope_bytes_per_sec"`
DaysTo90 float64 `json:"days_to_90_pct"`
DaysTo95 float64 `json:"days_to_95_pct"`
}
type CapacityForecastReport struct {
Cluster string `json:"cluster"`
Samples int `json:"samples"`
Items []CapacityForecastItem `json:"items,omitempty"`
}
func (s *Service) CapacityForecast(selector string, since time.Time) (CapacityForecastReport, error) {
c, err := s.Get(selector)
if err != nil {
return CapacityForecastReport{}, err
}
store := history.NewCapacityStore(s.DataDir())
snaps, err := store.Load(c.ID, since)
if err != nil {
return CapacityForecastReport{}, err
}
rep := CapacityForecastReport{Cluster: c.Name, Samples: len(snaps)}
if len(snaps) < 2 {
return rep, nil
}
type point struct {
ts time.Time
used float64
tot float64
}
byMount := map[string][]point{}
for _, snap := range snaps {
for _, d := range snap.Disks {
if d.TotalBytes == 0 {
continue
}
byMount[d.Mount] = append(byMount[d.Mount], point{ts: snap.Timestamp, used: float64(d.UsedBytes), tot: float64(d.TotalBytes)})
}
}
for mnt, pts := range byMount {
if len(pts) < 2 {
continue
}
sort.Slice(pts, func(i, j int) bool { return pts[i].ts.Before(pts[j].ts) })
first := pts[0]
last := pts[len(pts)-1]
dt := last.ts.Sub(first.ts).Seconds()
if dt <= 0 {
continue
}
slope := (last.used - first.used) / dt
usedPct := 100 * last.used / last.tot
d90 := daysToTarget(last.used, last.tot*0.90, slope)
d95 := daysToTarget(last.used, last.tot*0.95, slope)
rep.Items = append(rep.Items, CapacityForecastItem{
Mount: mnt,
UsedPct: usedPct,
SlopeBytesSec: slope,
DaysTo90: d90,
DaysTo95: d95,
})
}
sort.Slice(rep.Items, func(i, j int) bool { return rep.Items[i].UsedPct > rep.Items[j].UsedPct })
return rep, nil
}
func daysToTarget(current, target, slope float64) float64 {
if target <= current {
return 0
}
if slope <= 0 {
return math.Inf(1)
}
return (target - current) / slope / 86400
}
+425
View File
@@ -0,0 +1,425 @@
package cluster
import (
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
envConfigPath = "PXMON_CONFIG"
envMasterKeyPath = "PXMON_MASTER_KEY"
filePrefix = "OBSCTL1:"
masterKeyBytes = 32
)
// Store persists encrypted cluster registry on disk.
type Store struct {
path string
keyPath string
}
func NewStore(path string) (*Store, error) {
if path == "" {
var err error
path, err = DefaultConfigPath()
if err != nil {
return nil, err
}
}
keyPath, err := defaultMasterKeyPath(path)
if err != nil {
return nil, err
}
return &Store{path: path, keyPath: keyPath}, nil
}
func DefaultConfigPath() (string, error) {
if p := os.Getenv(envConfigPath); p != "" {
return p, nil
}
dir, err := os.UserConfigDir()
if err != nil {
return "", fmt.Errorf("resolve user config dir: %w", err)
}
return filepath.Join(dir, "pxmon", "clusters.enc"), nil
}
func defaultMasterKeyPath(configPath string) (string, error) {
if p := os.Getenv(envMasterKeyPath); p != "" {
return p, nil
}
if configPath == "" {
return "", errors.New("empty config path")
}
return filepath.Join(filepath.Dir(configPath), "master.key"), nil
}
func (s *Store) Path() string {
return s.path
}
func (s *Store) KeyPath() string {
return s.keyPath
}
func (s *Store) LockerSessionPath() string {
return filepath.Join(filepath.Dir(s.path), "locker.session")
}
func (s *Store) LockerAuditPath() string {
return filepath.Join(filepath.Dir(s.path), "locker.audit.log")
}
func (s *Store) Load() (Registry, error) {
f, err := os.Open(s.path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return newRegistry(), nil
}
return Registry{}, fmt.Errorf("open registry file: %w", err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
return Registry{}, fmt.Errorf("read registry file: %w", err)
}
if len(strings.TrimSpace(string(data))) == 0 {
return newRegistry(), nil
}
payload, err := s.decodePayload(data)
if err != nil {
return Registry{}, err
}
var reg Registry
if err := json.Unmarshal(payload, &reg); err != nil {
return Registry{}, fmt.Errorf("decode registry JSON: %w", err)
}
if reg.Version == 0 {
reg.Version = currentVersion
}
if reg.Clusters == nil {
reg.Clusters = []Cluster{}
}
for i := range reg.Clusters {
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
}
reg.Telegram = normalizeTelegram(reg.Telegram)
reg.Locker = normalizeLocker(reg.Locker)
reg.Backups = normalizeBackupConfig(reg.Backups)
return reg, nil
}
func (s *Store) decodePayload(data []byte) ([]byte, error) {
text := strings.TrimSpace(string(data))
if strings.HasPrefix(text, "{") {
// Backward compatibility with legacy unencrypted format.
return []byte(text), nil
}
if !strings.HasPrefix(text, filePrefix) {
return nil, errors.New("unsupported registry format")
}
blob := strings.TrimPrefix(text, filePrefix)
raw, err := base64.StdEncoding.DecodeString(blob)
if err != nil {
return nil, fmt.Errorf("decode encrypted payload: %w", err)
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return nil, err
}
payload, err := decrypt(raw, key)
if err != nil {
return nil, fmt.Errorf("decrypt registry: %w", err)
}
return payload, nil
}
func (s *Store) Save(reg Registry) error {
reg.Version = currentVersion
if reg.Clusters == nil {
reg.Clusters = []Cluster{}
}
for i := range reg.Clusters {
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
}
reg.Telegram = normalizeTelegram(reg.Telegram)
reg.Locker = normalizeLocker(reg.Locker)
reg.Backups = normalizeBackupConfig(reg.Backups)
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
return fmt.Errorf("create config dir: %w", err)
}
payload, err := json.MarshalIndent(reg, "", " ")
if err != nil {
return fmt.Errorf("encode registry JSON: %w", err)
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return err
}
encrypted, err := encrypt(payload, key)
if err != nil {
return fmt.Errorf("encrypt registry: %w", err)
}
content := filePrefix + base64.StdEncoding.EncodeToString(encrypted) + "\n"
tmp := s.path + ".tmp"
if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
return fmt.Errorf("write temp registry file: %w", err)
}
if err := os.Rename(tmp, s.path); err != nil {
return fmt.Errorf("replace registry file: %w", err)
}
return nil
}
func (s *Store) loadOrCreateMasterKey() ([]byte, error) {
if err := os.MkdirAll(filepath.Dir(s.keyPath), 0o700); err != nil {
return nil, fmt.Errorf("create key dir: %w", err)
}
key, err := os.ReadFile(s.keyPath)
if err == nil {
if len(key) != masterKeyBytes {
return nil, fmt.Errorf("invalid master key length: got %d", len(key))
}
return key, nil
}
if !errors.Is(err, os.ErrNotExist) {
return nil, fmt.Errorf("read master key: %w", err)
}
key = make([]byte, masterKeyBytes)
if _, err := rand.Read(key); err != nil {
return nil, fmt.Errorf("generate master key: %w", err)
}
tmp := s.keyPath + ".tmp"
if err := os.WriteFile(tmp, key, 0o600); err != nil {
return nil, fmt.Errorf("write temp master key: %w", err)
}
if err := os.Rename(tmp, s.keyPath); err != nil {
return nil, fmt.Errorf("replace master key: %w", err)
}
return key, nil
}
func encrypt(payload, key []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return nil, err
}
sealed := gcm.Seal(nil, nonce, payload, nil)
out := make([]byte, 0, len(nonce)+len(sealed))
out = append(out, nonce...)
out = append(out, sealed...)
return out, nil
}
func decrypt(raw, key []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
gcm, err := cipher.NewGCM(block)
if err != nil {
return nil, err
}
nonceSize := gcm.NonceSize()
if len(raw) <= nonceSize {
return nil, errors.New("ciphertext too short")
}
nonce := raw[:nonceSize]
ciphertext := raw[nonceSize:]
payload, err := gcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, err
}
return payload, nil
}
type lockerSessionState struct {
ExpiresAtUnix int64 `json:"expires_at_unix"`
SigHex string `json:"sig_hex"`
}
func (s *Store) SaveLockerSession(passwordHash string, ttl time.Duration) error {
passwordHash = strings.TrimSpace(passwordHash)
if passwordHash == "" {
return errors.New("empty locker password hash")
}
if ttl <= 0 {
ttl = 6 * time.Hour
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return err
}
expires := time.Now().UTC().Add(ttl).Unix()
payload := strconv.FormatInt(expires, 10) + "|" + passwordHash
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(payload))
sig := hex.EncodeToString(mac.Sum(nil))
state := lockerSessionState{
ExpiresAtUnix: expires,
SigHex: sig,
}
data, err := json.Marshal(state)
if err != nil {
return err
}
data = append(data, '\n')
path := s.LockerSessionPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0o600); err != nil {
return err
}
return os.Rename(tmp, path)
}
func (s *Store) ValidateLockerSession(passwordHash string) (bool, time.Time, error) {
passwordHash = strings.TrimSpace(passwordHash)
if passwordHash == "" {
return false, time.Time{}, nil
}
path := s.LockerSessionPath()
raw, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return false, time.Time{}, nil
}
return false, time.Time{}, err
}
var state lockerSessionState
if err := json.Unmarshal(raw, &state); err != nil {
return false, time.Time{}, nil
}
if state.ExpiresAtUnix <= 0 || strings.TrimSpace(state.SigHex) == "" {
return false, time.Time{}, nil
}
expiresAt := time.Unix(state.ExpiresAtUnix, 0).UTC()
if time.Now().UTC().After(expiresAt) {
return false, expiresAt, nil
}
key, err := s.loadOrCreateMasterKey()
if err != nil {
return false, time.Time{}, err
}
payload := strconv.FormatInt(state.ExpiresAtUnix, 10) + "|" + passwordHash
mac := hmac.New(sha256.New, key)
_, _ = mac.Write([]byte(payload))
expected := mac.Sum(nil)
got, err := hex.DecodeString(strings.TrimSpace(state.SigHex))
if err != nil {
return false, expiresAt, nil
}
if !hmac.Equal(expected, got) {
return false, expiresAt, nil
}
return true, expiresAt, nil
}
func (s *Store) ClearLockerSession() error {
path := s.LockerSessionPath()
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
func (s *Store) AppendLockerAudit(event, detail string) error {
path := s.LockerAuditPath()
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
return err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
if err != nil {
return err
}
defer f.Close()
ts := time.Now().UTC().Format(time.RFC3339)
event = strings.TrimSpace(event)
detail = strings.TrimSpace(detail)
if event == "" {
event = "event"
}
if detail == "" {
detail = "-"
}
_, err = fmt.Fprintf(f, "%s event=%s detail=%s\n", ts, event, strconv.Quote(detail))
return err
}
+110
View File
@@ -0,0 +1,110 @@
package cluster
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestStoreLoadMissing(t *testing.T) {
t.Parallel()
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
reg, err := store.Load()
if err != nil {
t.Fatalf("Load error: %v", err)
}
if reg.Version != currentVersion {
t.Fatalf("expected version %d, got %d", currentVersion, reg.Version)
}
if len(reg.Clusters) != 0 {
t.Fatalf("expected empty clusters, got %d", len(reg.Clusters))
}
}
func TestStoreSaveLoadRoundtrip(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "clusters.enc")
store, err := NewStore(path)
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
seed := newRegistry()
seed.ActiveClusterID = "clu_1"
seed.Clusters = []Cluster{{
ID: "clu_1",
Name: "prod",
Host: "10.0.0.10",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "secret123",
}}
if err := store.Save(seed); err != nil {
t.Fatalf("Save error: %v", err)
}
got, err := store.Load()
if err != nil {
t.Fatalf("Load error: %v", err)
}
if got.ActiveClusterID != "clu_1" {
t.Fatalf("active cluster mismatch: %s", got.ActiveClusterID)
}
if len(got.Clusters) != 1 || got.Clusters[0].Name != "prod" {
t.Fatalf("unexpected clusters: %+v", got.Clusters)
}
if got.Clusters[0].Password != "secret123" {
t.Fatalf("password mismatch after decrypt: %s", got.Clusters[0].Password)
}
}
func TestStorePersistsEncryptedPayload(t *testing.T) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "clusters.enc")
store, err := NewStore(path)
if err != nil {
t.Fatalf("NewStore error: %v", err)
}
reg := newRegistry()
reg.Clusters = []Cluster{{
ID: "clu_1",
Name: "sensitive-prod",
Host: "192.168.1.1",
Port: 22,
User: "root",
AuthMethod: AuthMethodPassword,
Password: "very-secret",
}}
if err := store.Save(reg); err != nil {
t.Fatalf("Save error: %v", err)
}
raw, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile error: %v", err)
}
if !strings.HasPrefix(string(raw), filePrefix) {
t.Fatalf("expected encrypted file prefix %q", filePrefix)
}
if strings.Contains(string(raw), "sensitive-prod") || strings.Contains(string(raw), "very-secret") {
t.Fatal("plaintext secrets leaked into encrypted file")
}
if _, err := os.Stat(store.KeyPath()); err != nil {
t.Fatalf("master key not created: %v", err)
}
}
+144
View File
@@ -0,0 +1,144 @@
package cluster
import (
"errors"
"sort"
"strings"
)
func (s *Service) AddClusterTags(selector string, tags []string) (Cluster, error) {
return s.updateClusterTags(selector, tags, true)
}
func (s *Service) RemoveClusterTags(selector string, tags []string) (Cluster, error) {
return s.updateClusterTags(selector, tags, false)
}
func (s *Service) updateClusterTags(selector string, tags []string, add bool) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
changes := make(map[string]struct{}, len(tags))
for _, t := range normalizeTagList(tags) {
changes[t] = struct{}{}
}
if len(changes) == 0 {
return c, errors.New("at least one non-empty tag is required")
}
current := make(map[string]struct{}, len(c.Tags))
for _, t := range normalizeTagList(c.Tags) {
current[t] = struct{}{}
}
if add {
for t := range changes {
current[t] = struct{}{}
}
} else {
for t := range changes {
delete(current, t)
}
}
out := make([]string, 0, len(current))
for t := range current {
out = append(out, t)
}
sort.Strings(out)
c.Tags = out
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("cluster.tags", c.Name, strings.Join(out, ","))
return c, nil
}
func (s *Service) ListClusterTags(selector string) ([]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
return normalizeTagList(c.Tags), nil
}
func (s *Service) AddKVMTag(selector, vm string, tags []string) (Cluster, error) {
return s.updateKVMTags(selector, vm, tags, true)
}
func (s *Service) RemoveKVMTag(selector, vm string, tags []string) (Cluster, error) {
return s.updateKVMTags(selector, vm, tags, false)
}
func (s *Service) updateKVMTags(selector, vm string, tags []string, add bool) (Cluster, error) {
vm = strings.TrimSpace(vm)
if vm == "" {
return Cluster{}, errors.New("vm name is required")
}
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
if c.KVMTags == nil {
c.KVMTags = map[string][]string{}
}
current := make(map[string]struct{}, len(c.KVMTags[vm]))
for _, t := range normalizeTagList(c.KVMTags[vm]) {
current[t] = struct{}{}
}
changes := normalizeTagList(tags)
if len(changes) == 0 {
return Cluster{}, errors.New("at least one non-empty tag is required")
}
if add {
for _, t := range changes {
current[t] = struct{}{}
}
} else {
for _, t := range changes {
delete(current, t)
}
}
out := make([]string, 0, len(current))
for t := range current {
out = append(out, t)
}
sort.Strings(out)
if len(out) == 0 {
delete(c.KVMTags, vm)
} else {
c.KVMTags[vm] = out
}
c.KVMTags = normalizeVMTagMap(c.KVMTags)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("kvm.tags", c.Name, vm+"="+strings.Join(out, ","))
return c, nil
}
func (s *Service) ListKVMTags(selector, vm string) (map[string][]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
out := make(map[string][]string, len(c.KVMTags))
for k, v := range c.KVMTags {
if vm != "" && !strings.EqualFold(strings.TrimSpace(vm), k) {
continue
}
out[k] = append([]string(nil), normalizeTagList(v)...)
}
return out, nil
}
+269
View File
@@ -0,0 +1,269 @@
package cluster
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"errors"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"time"
"golang.org/x/crypto/ssh"
)
// agentClient wraps an *http.Client pointed at the correct base URL for a
// cluster's agent (direct or SSH-tunneled).
type agentClient struct {
http *http.Client
target string
close func()
}
func (a *agentClient) Close() {
if a != nil && a.close != nil {
a.close()
}
}
// newAgentClient builds the right HTTP client for reaching a cluster's agent.
//
// For TransportDirect it returns a plain client talking to cluster.Host:port.
//
// For TransportIPFabric it reuses a cached ssh.Client from the service pool
// and returns a client whose Transport routes every TCP connection through
// ssh.Client.Dial to 127.0.0.1:port. The SSH connection stays pooled after
// Close() — only the HTTP transport's idle conns are released.
func (s *Service) newAgentClient(ctx context.Context, c Cluster, timeout time.Duration) (*agentClient, error) {
if c.Agent.Port == 0 {
return nil, errors.New("agent port is not set")
}
if timeout <= 0 {
timeout = 8 * time.Second
}
scheme := agentScheme(c)
tlsCfg, err := agentTLSConfig(c)
if err != nil {
return nil, err
}
switch normalizeTransport(c.Transport) {
case TransportIPFabric:
sshClient, err := s.acquireTunnelClient(ctx, c)
if err != nil {
return nil, err
}
tr := &http.Transport{
TLSClientConfig: tlsCfg,
DialContext: func(dctx context.Context, network, _ string) (net.Conn, error) {
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(c.Agent.Port))
return sshDialWithContext(dctx, sshClient, network, addr)
},
DisableKeepAlives: true,
IdleConnTimeout: 30 * time.Second,
ResponseHeaderTimeout: timeout,
ExpectContinueTimeout: 1 * time.Second,
}
httpClient := &http.Client{
Timeout: timeout,
Transport: tr,
}
return &agentClient{
http: httpClient,
target: scheme + "://127.0.0.1:" + strconv.Itoa(c.Agent.Port),
close: func() {
tr.CloseIdleConnections()
},
}, nil
default:
tr := &http.Transport{
TLSClientConfig: tlsCfg,
ResponseHeaderTimeout: timeout,
ExpectContinueTimeout: 1 * time.Second,
}
httpClient := &http.Client{Timeout: timeout, Transport: tr}
return &agentClient{
http: httpClient,
target: scheme + "://" + net.JoinHostPort(c.Host, strconv.Itoa(c.Agent.Port)),
close: func() {
tr.CloseIdleConnections()
},
}, nil
}
}
func agentScheme(c Cluster) string {
if c.Agent.TLSEnabled {
return "https"
}
return "http"
}
func agentTLSConfig(c Cluster) (*tls.Config, error) {
if !c.Agent.TLSEnabled {
return nil, nil
}
fp := strings.ToLower(strings.TrimSpace(c.Agent.TLSFingerprint))
if fp == "" {
return nil, errors.New("agent TLS is enabled but certificate fingerprint is missing")
}
fp = strings.ReplaceAll(fp, ":", "")
want, err := hex.DecodeString(fp)
if err != nil {
return nil, fmt.Errorf("invalid agent TLS fingerprint: %w", err)
}
return &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true, // verified via explicit fingerprint pinning below
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("agent TLS: peer certificate is missing")
}
sum := sha256.Sum256(rawCerts[0])
if len(want) != len(sum) {
return errors.New("agent TLS: fingerprint length mismatch")
}
if !hmacEqual(sum[:], want) {
return errors.New("agent TLS: fingerprint mismatch")
}
return nil
},
}, nil
}
func hmacEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
var v byte
for i := 0; i < len(a); i++ {
v |= a[i] ^ b[i]
}
return v == 0
}
// acquireTunnelClient returns a pooled ssh.Client for the cluster, creating
// one if necessary. Credential changes invalidate the cached entry via the
// fingerprint field. Dead clients are evicted lazily: when a DialContext
// through a stale client fails, the caller invokes CloseTunnelClient and the
// next acquire re-dials.
func (s *Service) acquireTunnelClient(ctx context.Context, c Cluster) (*ssh.Client, error) {
fp := credentialFingerprint(c)
s.tunnelMu.Lock()
entry, ok := s.tunnelPool[c.ID]
if ok && entry.fp != fp {
_ = entry.client.Close()
delete(s.tunnelPool, c.ID)
entry = nil
ok = false
}
if ok {
s.tunnelMu.Unlock()
return entry.client, nil
}
s.tunnelMu.Unlock()
client, err := s.dialSSH(ctx, c, "", "")
if err != nil {
return nil, err
}
s.tunnelMu.Lock()
if existing, ok := s.tunnelPool[c.ID]; ok && existing.fp == fp {
// Another goroutine won the race; drop ours.
s.tunnelMu.Unlock()
_ = client.Close()
return existing.client, nil
}
s.tunnelPool[c.ID] = &tunneledSSH{
client: client,
fp: fp,
}
s.tunnelMu.Unlock()
return client, nil
}
// CloseTunnelClient drops a pooled SSH tunnel for a cluster. Safe to call if
// no entry exists.
func (s *Service) CloseTunnelClient(clusterID string) {
s.tunnelMu.Lock()
entry, ok := s.tunnelPool[clusterID]
if ok {
delete(s.tunnelPool, clusterID)
}
s.tunnelMu.Unlock()
if ok && entry != nil && entry.client != nil {
_ = entry.client.Close()
}
}
// CloseAllTunnelClients tears down every pooled SSH tunnel.
func (s *Service) CloseAllTunnelClients() {
s.tunnelMu.Lock()
pool := s.tunnelPool
s.tunnelPool = make(map[string]*tunneledSSH)
s.tunnelMu.Unlock()
for _, e := range pool {
if e != nil && e.client != nil {
_ = e.client.Close()
}
}
}
// credentialFingerprint returns a short hash over the fields that affect how
// we'd reconnect. If any of these change we must not reuse a cached client.
func credentialFingerprint(c Cluster) string {
h := sha256.New()
h.Write([]byte(c.Host))
h.Write([]byte{'|'})
h.Write([]byte(strconv.Itoa(c.Port)))
h.Write([]byte{'|'})
h.Write([]byte(c.User))
h.Write([]byte{'|'})
h.Write([]byte(c.AuthMethod))
h.Write([]byte{'|'})
h.Write([]byte(c.Password))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPath))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPassphrase))
h.Write([]byte{'|'})
h.Write([]byte(c.KeyPassphraseFile))
return hex.EncodeToString(h.Sum(nil)[:8])
}
// sshDialWithContext wraps ssh.Client.Dial so it respects ctx cancellation.
// ssh.Client has no context-aware dial, so we fall back to a watcher goroutine
// that closes the connection if ctx fires before the dial returns.
func sshDialWithContext(ctx context.Context, client *ssh.Client, network, addr string) (net.Conn, error) {
type result struct {
conn net.Conn
err error
}
ch := make(chan result, 1)
go func() {
conn, err := client.Dial(network, addr)
ch <- result{conn: conn, err: err}
}()
select {
case <-ctx.Done():
go func() {
r := <-ch
if r.conn != nil {
_ = r.conn.Close()
}
}()
return nil, ctx.Err()
case r := <-ch:
return r.conn, r.err
}
}
+272
View File
@@ -0,0 +1,272 @@
package cluster
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"time"
"pxmon/internal/agent"
"pxmon/internal/history"
)
// UsageRange is a façade over history.RangeShortcut so callers outside this
// package don't need to import both.
type UsageRange = history.RangeShortcut
// UsageSnapshot bundles everything the "usage" page shows for a cluster at a
// single point in time.
type UsageSnapshot struct {
ClusterID string `json:"cluster_id"`
ClusterName string `json:"cluster_name"`
Range string `json:"range"`
GeneratedAt time.Time `json:"generated_at"`
Live agent.StatsResponse `json:"live"`
Top agent.TopResponse `json:"top"`
DU agent.DUResponse `json:"du"`
NodeSeries []history.NodeSamplePoint `json:"series,omitempty"`
P95TotalMbps float64 `json:"p95_total_mbps"`
MaxTotalMbps float64 `json:"max_total_mbps"`
AvgTotalMbps float64 `json:"avg_total_mbps"`
TopIfaceName string `json:"top_iface_name,omitempty"`
TopIfaceMbps float64 `json:"top_iface_mbps,omitempty"`
DUError string `json:"du_error,omitempty"`
TopError string `json:"top_error,omitempty"`
HistoryError string `json:"history_error,omitempty"`
}
// AgentTopProcesses calls /api/v1/top on the selected cluster's agent.
func (s *Service) AgentTopProcesses(ctx context.Context, selector string, sampleWindow time.Duration, limit int) (agent.TopResponse, error) {
cluster, err := s.Get(selector)
if err != nil {
return agent.TopResponse{}, err
}
if !cluster.Agent.Installed {
return agent.TopResponse{}, errors.New("agent is not installed on this cluster")
}
ac, err := s.newAgentClient(ctx, cluster, 15*time.Second)
if err != nil {
return agent.TopResponse{}, err
}
defer ac.Close()
url := ac.target + "/api/v1/top"
q := ""
if sampleWindow > 0 {
q += "sample_ms=" + strconv.FormatInt(sampleWindow.Milliseconds(), 10)
}
if limit > 0 {
if q != "" {
q += "&"
}
q += "limit=" + strconv.Itoa(limit)
}
if q != "" {
url += "?" + q
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return agent.TopResponse{}, err
}
applyAgentRequestAuth(req, cluster)
resp, err := ac.http.Do(req)
if err != nil {
if normalizeTransport(cluster.Transport) == TransportIPFabric {
s.CloseTunnelClient(cluster.ID)
}
return agent.TopResponse{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return agent.TopResponse{}, err
}
if resp.StatusCode >= 300 {
msg := string(body)
if len(msg) > 200 {
msg = msg[:200]
}
return agent.TopResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
}
var out agent.TopResponse
if err := json.Unmarshal(body, &out); err != nil {
return agent.TopResponse{}, err
}
return out, nil
}
// AgentDirSizes calls /api/v1/du on the selected cluster's agent.
func (s *Service) AgentDirSizes(ctx context.Context, selector, path string, limit int, timeout time.Duration) (agent.DUResponse, error) {
cluster, err := s.Get(selector)
if err != nil {
return agent.DUResponse{}, err
}
if !cluster.Agent.Installed {
return agent.DUResponse{}, errors.New("agent is not installed on this cluster")
}
httpTimeout := timeout + 10*time.Second
if httpTimeout < 20*time.Second {
httpTimeout = 20 * time.Second
}
ac, err := s.newAgentClient(ctx, cluster, httpTimeout)
if err != nil {
return agent.DUResponse{}, err
}
defer ac.Close()
url := ac.target + "/api/v1/du"
q := ""
if path != "" {
q += "path=" + path
}
if limit > 0 {
if q != "" {
q += "&"
}
q += "limit=" + strconv.Itoa(limit)
}
if timeout > 0 {
if q != "" {
q += "&"
}
q += "timeout_ms=" + strconv.FormatInt(timeout.Milliseconds(), 10)
}
if q != "" {
url += "?" + q
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return agent.DUResponse{}, err
}
applyAgentRequestAuth(req, cluster)
resp, err := ac.http.Do(req)
if err != nil {
if normalizeTransport(cluster.Transport) == TransportIPFabric {
s.CloseTunnelClient(cluster.ID)
}
return agent.DUResponse{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return agent.DUResponse{}, err
}
if resp.StatusCode >= 300 {
msg := string(body)
if len(msg) > 200 {
msg = msg[:200]
}
return agent.DUResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
}
var out agent.DUResponse
if err := json.Unmarshal(body, &out); err != nil {
return agent.DUResponse{}, err
}
return out, nil
}
// CollectUsageSnapshot gathers every data source needed by the usage page:
// live stats, top processes, top folders, and historical aggregate for P95.
// Errors in individual components are recorded on the snapshot instead of
// aborting the whole call — the page degrades gracefully.
func (s *Service) CollectUsageSnapshot(ctx context.Context, selector string, rng UsageRange, duPath string) (UsageSnapshot, error) {
cluster, err := s.Get(selector)
if err != nil {
return UsageSnapshot{}, err
}
snap := UsageSnapshot{
ClusterID: cluster.ID,
ClusterName: cluster.Name,
Range: string(rng),
GeneratedAt: time.Now().UTC(),
}
if live, err := s.AgentStatsTyped(ctx, selector); err != nil {
return snap, fmt.Errorf("live stats: %w", err)
} else {
snap.Live = live
}
if top, err := s.AgentTopProcesses(ctx, selector, 300*time.Millisecond, 15); err != nil {
snap.TopError = err.Error()
} else {
snap.Top = top
}
if duPath == "" {
duPath = "/"
}
if du, err := s.AgentDirSizes(ctx, selector, duPath, 12, 12*time.Second); err != nil {
snap.DUError = err.Error()
} else {
snap.DU = du
}
// Network history + P95 aggregation.
if s.networkStore == nil {
snap.HistoryError = "history store not configured"
} else {
since := history.RangeShortcut(rng).Since(time.Now())
if rng == "" || rng == history.RangeLive {
since = time.Now().Add(-5 * time.Minute)
}
snapshots, err := s.networkStore.Load(cluster.ID, since)
if err != nil {
snap.HistoryError = err.Error()
} else {
// Pick the physical uplink once and use it consistently for
// the series, P95, and the "top iface" display so they all
// describe the same thing.
primary := history.PrimaryInterface(snapshots)
series := history.AggregateNodeSeries(snapshots, primary)
snap.NodeSeries = series
snap.P95TotalMbps = history.PercentileMbps(series, 95)
if len(series) > 0 {
maxV := 0.0
sum := 0.0
for _, p := range series {
if p.TotalMbps > maxV {
maxV = p.TotalMbps
}
sum += p.TotalMbps
}
snap.MaxTotalMbps = maxV
snap.AvgTotalMbps = sum / float64(len(series))
}
snap.TopIfaceName = primary
snap.TopIfaceMbps = snap.AvgTotalMbps
}
}
return snap, nil
}
// RenderUsageChartPNG is a thin helper that reads a usage snapshot's series
// and delegates to history.RenderNodeNetworkPNG.
func RenderUsageChartPNG(snap UsageSnapshot, title string) ([]byte, error) {
if title == "" {
title = fmt.Sprintf("%s — node network usage (%s)", snap.ClusterName, snap.Range)
}
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps",
snap.P95TotalMbps, snap.MaxTotalMbps, snap.AvgTotalMbps)
return history.RenderNodeNetworkPNG(snap.NodeSeries, history.ChartOptions{
Title: title,
Subtitle: subtitle,
Percentile: 95,
})
}
+137
View File
@@ -0,0 +1,137 @@
package cluster
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
type VMStateSummary struct {
Total int `json:"total"`
Running int `json:"running"`
ShutOff int `json:"shut_off"`
Paused int `json:"paused"`
Others int `json:"others"`
ShutOffNames []string `json:"shut_off_names,omitempty"`
PausedNames []string `json:"paused_names,omitempty"`
OtherNames []string `json:"other_names,omitempty"`
Warnings []string `json:"warnings,omitempty"`
SampledAt time.Time `json:"sampled_at"`
}
func (s *Service) GetVMAlertPolicy(selector string) (VMAlertPolicy, error) {
c, err := s.Get(selector)
if err != nil {
return VMAlertPolicy{}, err
}
return ensureVMAlertPolicy(c.VMAlerts), nil
}
func (s *Service) SetVMAlertPolicy(selector string, p VMAlertPolicy) (Cluster, error) {
reg, err := s.store.Load()
if err != nil {
return Cluster{}, err
}
c, idx, err := findCluster(reg, selector)
if err != nil {
return Cluster{}, err
}
c.VMAlerts = ensureVMAlertPolicy(p)
c.UpdatedAt = s.now().UTC()
reg.Clusters[idx] = c
if err := s.store.Save(reg); err != nil {
return Cluster{}, err
}
_ = s.AppendChange("vm.alerts", c.Name, fmt.Sprintf("enabled=%t warn_on_shutoff=%t min_running=%d", c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning))
return c, nil
}
func (s *Service) CheckVMAlerts(ctx context.Context, selector string) (VMStateSummary, error) {
c, err := s.Get(selector)
if err != nil {
return VMStateSummary{}, err
}
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
if err != nil {
return VMStateSummary{}, err
}
states := parseVirshListStates(out)
summary := VMStateSummary{SampledAt: s.now().UTC(), Total: len(states)}
for name, st := range states {
n := strings.ToLower(strings.TrimSpace(st))
switch {
case n == "running":
summary.Running++
case n == "shut off":
summary.ShutOff++
summary.ShutOffNames = append(summary.ShutOffNames, name)
case n == "paused":
summary.Paused++
summary.PausedNames = append(summary.PausedNames, name)
default:
summary.Others++
summary.OtherNames = append(summary.OtherNames, name)
}
}
sort.Strings(summary.ShutOffNames)
sort.Strings(summary.PausedNames)
sort.Strings(summary.OtherNames)
p := ensureVMAlertPolicy(c.VMAlerts)
if p.Enabled {
if p.WarnOnShutoff && summary.ShutOff > 0 {
summary.Warnings = append(summary.Warnings, fmt.Sprintf(
"%d VM(s) are shut off: %s",
summary.ShutOff,
joinNamesLimit(summary.ShutOffNames, 12),
))
}
if summary.Running < p.MinRunning {
summary.Warnings = append(summary.Warnings, fmt.Sprintf("running VM count %d is below min_running=%d", summary.Running, p.MinRunning))
}
}
sort.Strings(summary.Warnings)
return summary, nil
}
func (s *Service) ListVMStates(ctx context.Context, selector string) (map[string]string, error) {
c, err := s.Get(selector)
if err != nil {
return nil, err
}
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
if err != nil {
return nil, err
}
return parseVirshListStates(out), nil
}
func parseVirshListStates(raw string) map[string]string {
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
out := make(map[string]string)
for _, line := range lines {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "Id") || strings.HasPrefix(line, "-") {
continue
}
fields := strings.Fields(line)
if len(fields) < 3 {
continue
}
name := fields[1]
state := strings.Join(fields[2:], " ")
out[name] = state
}
return out
}
func joinNamesLimit(items []string, limit int) string {
if len(items) == 0 {
return "-"
}
if limit <= 0 || len(items) <= limit {
return strings.Join(items, ", ")
}
return strings.Join(items[:limit], ", ") + fmt.Sprintf(" (+%d more)", len(items)-limit)
}