chore: publish pxmon v0.2.0
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
agentHeaderTS = "X-Observer-Ts"
|
||||
agentHeaderNonce = "X-Observer-Nonce"
|
||||
agentHeaderSignature = "X-Observer-Signature"
|
||||
)
|
||||
|
||||
func applyAgentRequestAuth(req *http.Request, c Cluster) {
|
||||
if req == nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(c.Agent.Token) != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.Agent.Token)
|
||||
}
|
||||
secret := strings.TrimSpace(c.Agent.RequestSecret)
|
||||
if secret == "" {
|
||||
return
|
||||
}
|
||||
|
||||
ts := strconv.FormatInt(time.Now().UTC().Unix(), 10)
|
||||
nonce := randomHex(12)
|
||||
payload := req.Method + "\n" + req.URL.RequestURI() + "\n" + ts + "\n" + nonce
|
||||
|
||||
mac := hmac.New(sha256.New, []byte(secret))
|
||||
_, _ = mac.Write([]byte(payload))
|
||||
sig := hex.EncodeToString(mac.Sum(nil))
|
||||
|
||||
req.Header.Set(agentHeaderTS, ts)
|
||||
req.Header.Set(agentHeaderNonce, nonce)
|
||||
req.Header.Set(agentHeaderSignature, sig)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//go:embed agent_versions.json
|
||||
var agentVersionsFS embed.FS
|
||||
|
||||
type AgentVersionInfo struct {
|
||||
Version string `json:"version"`
|
||||
ReleasedAt string `json:"released_at,omitempty"`
|
||||
Features []string `json:"features,omitempty"`
|
||||
}
|
||||
|
||||
func loadAgentVersions() []AgentVersionInfo {
|
||||
raw, err := agentVersionsFS.ReadFile("agent_versions.json")
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var items []AgentVersionInfo
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]AgentVersionInfo, 0, len(items))
|
||||
for _, it := range items {
|
||||
it.Version = strings.TrimSpace(it.Version)
|
||||
if it.Version == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) AgentVersions() []AgentVersionInfo {
|
||||
return append([]AgentVersionInfo(nil), loadAgentVersions()...)
|
||||
}
|
||||
|
||||
func (s *Service) AgentVersionFeatures(version string) []string {
|
||||
v := strings.TrimSpace(version)
|
||||
if v == "" {
|
||||
return nil
|
||||
}
|
||||
for _, it := range loadAgentVersions() {
|
||||
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
|
||||
return append([]string(nil), it.Features...)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) CompareAgentVersion(version string) (isLatest bool, latest string, known bool) {
|
||||
items := loadAgentVersions()
|
||||
if len(items) == 0 {
|
||||
return true, "", false
|
||||
}
|
||||
latest = strings.TrimSpace(items[len(items)-1].Version)
|
||||
v := strings.TrimSpace(version)
|
||||
for _, it := range items {
|
||||
if strings.EqualFold(strings.TrimSpace(it.Version), v) {
|
||||
return strings.EqualFold(v, latest), latest, true
|
||||
}
|
||||
}
|
||||
return false, latest, false
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
[
|
||||
{
|
||||
"version": "dev",
|
||||
"released_at": "2026-04-10",
|
||||
"features": [
|
||||
"cluster usage/traffic/graph",
|
||||
"kvm top static spec",
|
||||
"telegram graph sendPhoto fallback"
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "dev-2026.04.17",
|
||||
"released_at": "2026-04-17",
|
||||
"features": [
|
||||
"cluster p95 by interface",
|
||||
"cluster tag + kvm tag",
|
||||
"cluster vm alert-rules",
|
||||
"cluster drift",
|
||||
"cluster runbook",
|
||||
"cluster scheduler",
|
||||
"cluster change-history",
|
||||
"cluster report export"
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "v0.2.0",
|
||||
"released_at": "2026-06-16",
|
||||
"features": [
|
||||
"repo tunneling gateway/proxy workflows",
|
||||
"cluster exec/run commands",
|
||||
"ssh key passphrase file support",
|
||||
"export/import referenced key files",
|
||||
"signed agent request skew hardening",
|
||||
"tui refresh and network dashboard improvements"
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,655 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/pkg/sftp"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
type BackupRunResult struct {
|
||||
PlanID string `json:"plan_id"`
|
||||
PlanName string `json:"plan_name"`
|
||||
TargetID string `json:"target_id"`
|
||||
TargetName string `json:"target_name"`
|
||||
ArchiveName string `json:"archive_name"`
|
||||
UploadedTo string `json:"uploaded_to"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
RanAt time.Time `json:"ran_at"`
|
||||
}
|
||||
|
||||
func (s *Service) BackupListTargets() ([]BackupTarget, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupListPlans() ([]BackupPlan, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
selector = strings.TrimSpace(selector)
|
||||
var target *BackupTarget
|
||||
for i := range cfg.Targets {
|
||||
if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) {
|
||||
target = &cfg.Targets[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
return "", fmt.Errorf("backup target not found")
|
||||
}
|
||||
switch target.Type {
|
||||
case "sftp":
|
||||
host := strings.TrimSpace(target.SFTPHost)
|
||||
user := strings.TrimSpace(target.SFTPUser)
|
||||
if host == "" || user == "" {
|
||||
return "", fmt.Errorf("sftp target has empty host/user")
|
||||
}
|
||||
addr := netJoinHostPort(host, target.SFTPPort)
|
||||
auths := make([]ssh.AuthMethod, 0, 2)
|
||||
if strings.TrimSpace(target.SFTPPassword) != "" {
|
||||
auths = append(auths, ssh.Password(target.SFTPPassword))
|
||||
}
|
||||
if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" {
|
||||
pemBytes, err := os.ReadFile(kp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read sftp key: %w", err)
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(pemBytes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse sftp key: %w", err)
|
||||
}
|
||||
auths = append(auths, ssh.PublicKeys(signer))
|
||||
}
|
||||
if len(auths) == 0 {
|
||||
return "", fmt.Errorf("sftp auth is required (password or key)")
|
||||
}
|
||||
sshCfg := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: auths,
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
|
||||
Timeout: 15 * time.Second,
|
||||
}
|
||||
conn, err := ssh.Dial("tcp", addr, sshCfg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer conn.Close()
|
||||
c, err := sftp.NewClient(conn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer c.Close()
|
||||
base := strings.TrimSpace(target.SFTPBasePath)
|
||||
if base == "" {
|
||||
base = "."
|
||||
}
|
||||
if err := c.MkdirAll(base); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := c.ReadDir(base); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil
|
||||
case "s3":
|
||||
endpoint := strings.TrimSpace(target.S3Endpoint)
|
||||
bucket := strings.TrimSpace(target.S3Bucket)
|
||||
access := strings.TrimSpace(target.S3AccessKey)
|
||||
secret := strings.TrimSpace(target.S3SecretKey)
|
||||
if endpoint == "" || bucket == "" || access == "" || secret == "" {
|
||||
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
|
||||
}
|
||||
region := strings.TrimSpace(target.S3Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
lookup := minio.BucketLookupAuto
|
||||
if target.S3PathStyle {
|
||||
lookup = minio.BucketLookupPath
|
||||
}
|
||||
cli, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""),
|
||||
Secure: target.S3UseSSL,
|
||||
Region: region,
|
||||
BucketLookup: lookup,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
exists, err := cli.BucketExists(ctx, bucket)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !exists {
|
||||
return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket)
|
||||
}
|
||||
scheme := "https"
|
||||
if !target.S3UseSSL {
|
||||
scheme = "http"
|
||||
}
|
||||
return scheme + "://" + endpoint + "/" + bucket, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported target type %q", target.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return BackupTarget{}, err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
t.ID = strings.TrimSpace(t.ID)
|
||||
if t.ID == "" {
|
||||
t.ID = newClusterID()
|
||||
}
|
||||
t.Name = strings.TrimSpace(t.Name)
|
||||
if t.Name == "" {
|
||||
return BackupTarget{}, fmt.Errorf("target name is required")
|
||||
}
|
||||
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
|
||||
if t.Type != "sftp" && t.Type != "s3" {
|
||||
return BackupTarget{}, fmt.Errorf("target type must be sftp|s3")
|
||||
}
|
||||
if t.Type == "sftp" {
|
||||
if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" {
|
||||
return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user")
|
||||
}
|
||||
if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" {
|
||||
return BackupTarget{}, fmt.Errorf("sftp target requires password or key")
|
||||
}
|
||||
}
|
||||
if t.Type == "s3" {
|
||||
if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" {
|
||||
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket")
|
||||
}
|
||||
if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" {
|
||||
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key")
|
||||
}
|
||||
}
|
||||
for _, ex := range cfg.Targets {
|
||||
if strings.EqualFold(ex.Name, t.Name) {
|
||||
return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name)
|
||||
}
|
||||
}
|
||||
now := s.now().UTC()
|
||||
t.CreatedAt = now
|
||||
t.UpdatedAt = now
|
||||
if t.SFTPPort <= 0 {
|
||||
t.SFTPPort = 22
|
||||
}
|
||||
if !t.Enabled {
|
||||
t.Enabled = true
|
||||
}
|
||||
cfg.Targets = append(cfg.Targets, t)
|
||||
reg.Backups = cfg
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return BackupTarget{}, err
|
||||
}
|
||||
_ = s.AppendChange("backup.target.add", t.ID, t.Name)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return BackupTarget{}, err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
selector = strings.TrimSpace(selector)
|
||||
idx := -1
|
||||
for i, t := range cfg.Targets {
|
||||
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return BackupTarget{}, fmt.Errorf("backup target not found")
|
||||
}
|
||||
removed := cfg.Targets[idx]
|
||||
cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...)
|
||||
reg.Backups = cfg
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return BackupTarget{}, err
|
||||
}
|
||||
_ = s.AppendChange("backup.target.remove", removed.ID, removed.Name)
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return BackupPlan{}, err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
if strings.TrimSpace(p.Name) == "" {
|
||||
return BackupPlan{}, fmt.Errorf("plan name is required")
|
||||
}
|
||||
if strings.TrimSpace(p.TargetID) == "" {
|
||||
return BackupPlan{}, fmt.Errorf("target is required")
|
||||
}
|
||||
if len(p.Paths) == 0 {
|
||||
return BackupPlan{}, fmt.Errorf("at least one path is required")
|
||||
}
|
||||
targetID := ""
|
||||
for _, t := range cfg.Targets {
|
||||
if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) {
|
||||
targetID = t.ID
|
||||
break
|
||||
}
|
||||
}
|
||||
if targetID == "" {
|
||||
return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID)
|
||||
}
|
||||
for _, ex := range cfg.Plans {
|
||||
if strings.EqualFold(ex.Name, p.Name) {
|
||||
return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name)
|
||||
}
|
||||
}
|
||||
p.ID = newClusterID()
|
||||
p.TargetID = targetID
|
||||
p.Paths = normalizeBackupPaths(p.Paths)
|
||||
if strings.TrimSpace(p.Every) == "" {
|
||||
p.Every = "24h"
|
||||
}
|
||||
if p.RetainDays <= 0 {
|
||||
p.RetainDays = 30
|
||||
}
|
||||
p.Compress = true
|
||||
now := s.now().UTC()
|
||||
p.CreatedAt = now
|
||||
p.UpdatedAt = now
|
||||
if !p.Enabled {
|
||||
p.Enabled = true
|
||||
}
|
||||
cfg.Plans = append(cfg.Plans, p)
|
||||
reg.Backups = cfg
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return BackupPlan{}, err
|
||||
}
|
||||
_ = s.AppendChange("backup.plan.add", p.ID, p.Name)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return BackupPlan{}, err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
selector = strings.TrimSpace(selector)
|
||||
idx := -1
|
||||
for i, p := range cfg.Plans {
|
||||
if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return BackupPlan{}, fmt.Errorf("backup plan not found")
|
||||
}
|
||||
removed := cfg.Plans[idx]
|
||||
cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
|
||||
reg.Backups = cfg
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return BackupPlan{}, err
|
||||
}
|
||||
_ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name)
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return BackupRunResult{}, err
|
||||
}
|
||||
cfg := normalizeBackupConfig(reg.Backups)
|
||||
|
||||
var plan *BackupPlan
|
||||
for i := range cfg.Plans {
|
||||
if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) {
|
||||
plan = &cfg.Plans[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if plan == nil {
|
||||
return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector)
|
||||
}
|
||||
var target *BackupTarget
|
||||
for i := range cfg.Targets {
|
||||
if cfg.Targets[i].ID == plan.TargetID {
|
||||
target = &cfg.Targets[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if target == nil {
|
||||
return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID)
|
||||
}
|
||||
|
||||
c, err := s.Get(plan.Cluster)
|
||||
if err != nil {
|
||||
return BackupRunResult{}, err
|
||||
}
|
||||
sshClient, err := s.dialSSH(ctx, c, "", "")
|
||||
if err != nil {
|
||||
return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err)
|
||||
}
|
||||
defer sshClient.Close()
|
||||
|
||||
ts := s.now().UTC().Format("20060102T150405Z")
|
||||
archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz"
|
||||
tmpPath := filepath.Join(os.TempDir(), archiveName)
|
||||
tmpFile, err := os.Create(tmpPath)
|
||||
if err != nil {
|
||||
return BackupRunResult{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = tmpFile.Close()
|
||||
_ = os.Remove(tmpPath)
|
||||
}()
|
||||
|
||||
remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths)
|
||||
if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil {
|
||||
plan.LastRunAt = s.now().UTC()
|
||||
plan.LastStatus = "failed"
|
||||
plan.LastError = err.Error()
|
||||
plan.UpdatedAt = s.now().UTC()
|
||||
reg.Backups = cfg
|
||||
_ = s.store.Save(reg)
|
||||
return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err)
|
||||
}
|
||||
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
|
||||
return BackupRunResult{}, err
|
||||
}
|
||||
st, _ := tmpFile.Stat()
|
||||
size := int64(0)
|
||||
if st != nil {
|
||||
size = st.Size()
|
||||
}
|
||||
|
||||
uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size)
|
||||
if err != nil {
|
||||
plan.LastRunAt = s.now().UTC()
|
||||
plan.LastStatus = "failed"
|
||||
plan.LastError = err.Error()
|
||||
plan.UpdatedAt = s.now().UTC()
|
||||
reg.Backups = cfg
|
||||
_ = s.store.Save(reg)
|
||||
return BackupRunResult{}, fmt.Errorf("upload backup: %w", err)
|
||||
}
|
||||
|
||||
plan.LastRunAt = s.now().UTC()
|
||||
plan.LastStatus = "ok"
|
||||
plan.LastError = ""
|
||||
plan.LastArchive = archiveName
|
||||
plan.UpdatedAt = s.now().UTC()
|
||||
reg.Backups = cfg
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return BackupRunResult{}, err
|
||||
}
|
||||
_ = s.AppendChange("backup.plan.run", plan.ID, archiveName)
|
||||
|
||||
return BackupRunResult{
|
||||
PlanID: plan.ID,
|
||||
PlanName: plan.Name,
|
||||
TargetID: target.ID,
|
||||
TargetName: target.Name,
|
||||
ArchiveName: archiveName,
|
||||
UploadedTo: uploadedTo,
|
||||
SizeBytes: size,
|
||||
RanAt: plan.LastRunAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeBackupPaths(in []string) []string {
|
||||
out := make([]string, 0, len(in))
|
||||
seen := map[string]struct{}{}
|
||||
for _, p := range in {
|
||||
v := strings.TrimSpace(p)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[v]; ok {
|
||||
continue
|
||||
}
|
||||
seen[v] = struct{}{}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sanitizeBackupName(v string) string {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
if v == "" {
|
||||
return "backup"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range v {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('-')
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), "-")
|
||||
if out == "" {
|
||||
return "backup"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildRemoteTarStreamCommand(paths []string) string {
|
||||
items := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
v := strings.TrimSpace(p)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, shellQuote(v))
|
||||
}
|
||||
if len(items) == 0 {
|
||||
items = []string{shellQuote("/")}
|
||||
}
|
||||
return "tar -czf - " + strings.Join(items, " ")
|
||||
}
|
||||
|
||||
func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error {
|
||||
session, err := client.NewSession()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer session.Close()
|
||||
stdout, err := session.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stderr, err := session.StderrPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
|
||||
return err
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, cpErr := io.Copy(w, stdout)
|
||||
if cpErr != nil {
|
||||
done <- cpErr
|
||||
return
|
||||
}
|
||||
done <- session.Wait()
|
||||
}()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = session.Close()
|
||||
return ctx.Err()
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
b, _ := io.ReadAll(stderr)
|
||||
msg := strings.TrimSpace(string(b))
|
||||
if msg == "" {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %s", err, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(target.Type)) {
|
||||
case "sftp":
|
||||
return uploadBackupSFTP(ctx, target, archiveName, r)
|
||||
case "s3":
|
||||
return uploadBackupS3(ctx, target, archiveName, r, size)
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported backup target type %q", target.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) {
|
||||
_ = ctx
|
||||
host := strings.TrimSpace(t.SFTPHost)
|
||||
if host == "" {
|
||||
return "", fmt.Errorf("sftp_host is required")
|
||||
}
|
||||
user := strings.TrimSpace(t.SFTPUser)
|
||||
if user == "" {
|
||||
return "", fmt.Errorf("sftp_user is required")
|
||||
}
|
||||
addr := netJoinHostPort(host, t.SFTPPort)
|
||||
auths := make([]ssh.AuthMethod, 0, 2)
|
||||
if strings.TrimSpace(t.SFTPPassword) != "" {
|
||||
auths = append(auths, ssh.Password(t.SFTPPassword))
|
||||
}
|
||||
if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" {
|
||||
pemBytes, err := os.ReadFile(kp)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read sftp key: %w", err)
|
||||
}
|
||||
signer, err := ssh.ParsePrivateKey(pemBytes)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse sftp key: %w", err)
|
||||
}
|
||||
auths = append(auths, ssh.PublicKeys(signer))
|
||||
}
|
||||
if len(auths) == 0 {
|
||||
return "", fmt.Errorf("sftp auth is required (password or key)")
|
||||
}
|
||||
sshCfg := &ssh.ClientConfig{
|
||||
User: user,
|
||||
Auth: auths,
|
||||
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust
|
||||
Timeout: 15 * time.Second,
|
||||
}
|
||||
conn, err := ssh.Dial("tcp", addr, sshCfg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer conn.Close()
|
||||
c, err := sftp.NewClient(conn)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
base := strings.TrimSpace(t.SFTPBasePath)
|
||||
if base == "" {
|
||||
base = "."
|
||||
}
|
||||
if err := c.MkdirAll(base); err != nil {
|
||||
return "", err
|
||||
}
|
||||
remote := path.Join(base, archiveName)
|
||||
f, err := c.Create(remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := io.Copy(f, r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil
|
||||
}
|
||||
|
||||
func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
|
||||
endpoint := strings.TrimSpace(t.S3Endpoint)
|
||||
bucket := strings.TrimSpace(t.S3Bucket)
|
||||
access := strings.TrimSpace(t.S3AccessKey)
|
||||
secret := strings.TrimSpace(t.S3SecretKey)
|
||||
if endpoint == "" || bucket == "" || access == "" || secret == "" {
|
||||
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
|
||||
}
|
||||
region := strings.TrimSpace(t.S3Region)
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
lookup := minio.BucketLookupAuto
|
||||
if t.S3PathStyle {
|
||||
lookup = minio.BucketLookupPath
|
||||
}
|
||||
cli, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""),
|
||||
Secure: t.S3UseSSL,
|
||||
Region: region,
|
||||
BucketLookup: lookup,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
key := archiveName
|
||||
if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" {
|
||||
key = p + "/" + archiveName
|
||||
}
|
||||
opts := minio.PutObjectOptions{ContentType: "application/gzip"}
|
||||
if size < 0 {
|
||||
size = -1
|
||||
}
|
||||
_, err = cli.PutObject(ctx, bucket, key, r, size, opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
scheme := "https"
|
||||
if !t.S3UseSSL {
|
||||
scheme = "http"
|
||||
}
|
||||
return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil
|
||||
}
|
||||
|
||||
func netJoinHostPort(host string, port int) string {
|
||||
p := port
|
||||
if p <= 0 {
|
||||
p = 22
|
||||
}
|
||||
return net.JoinHostPort(host, strconv.Itoa(p))
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ChangeRecord struct {
|
||||
At time.Time `json:"at"`
|
||||
Action string `json:"action"`
|
||||
Target string `json:"target"`
|
||||
Details string `json:"details,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) changeHistoryPath() string {
|
||||
return filepath.Join(s.DataDir(), "history", "changes.log")
|
||||
}
|
||||
|
||||
func (s *Service) AppendChange(action, target, details string) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
rec := ChangeRecord{
|
||||
At: s.now().UTC(),
|
||||
Action: strings.TrimSpace(action),
|
||||
Target: strings.TrimSpace(target),
|
||||
Details: strings.TrimSpace(details),
|
||||
}
|
||||
if rec.Action == "" {
|
||||
return nil
|
||||
}
|
||||
path := s.changeHistoryPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
b, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.Write(append(b, '\n'))
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Service) ListChanges(limit int) ([]ChangeRecord, error) {
|
||||
path := s.changeHistoryPath()
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
capHint := 32
|
||||
if limit > capHint {
|
||||
capHint = limit
|
||||
}
|
||||
items := make([]ChangeRecord, 0, capHint)
|
||||
sc := bufio.NewScanner(f)
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 2*1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var rec ChangeRecord
|
||||
if err := json.Unmarshal([]byte(line), &rec); err != nil {
|
||||
continue
|
||||
}
|
||||
items = append(items, rec)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, fmt.Errorf("scan change history: %w", err)
|
||||
}
|
||||
if limit > 0 && len(items) > limit {
|
||||
items = items[len(items)-limit:]
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type DriftIssue struct {
|
||||
Level string `json:"level"`
|
||||
Kind string `json:"kind"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
type DriftReport struct {
|
||||
Cluster string `json:"cluster"`
|
||||
Generated time.Time `json:"generated_at"`
|
||||
Issues []DriftIssue `json:"issues,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) DetectDrift(ctx context.Context, selector string) (DriftReport, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return DriftReport{}, err
|
||||
}
|
||||
rep := DriftReport{Cluster: c.Name, Generated: s.now().UTC()}
|
||||
expected := strings.TrimSpace(s.ExpectedAgentVersion())
|
||||
dctrl := normalizeDriftControl(c.Drift)
|
||||
|
||||
if !c.Agent.Installed {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent", Message: "agent is not installed"})
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 2500*time.Millisecond)
|
||||
ping, pingErr := s.PingAgent(pingCtx, c.ID)
|
||||
cancel()
|
||||
if pingErr != nil || !ping.Reachable || ping.StatusCode >= 400 {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "crit", Kind: "agent", Message: "agent is unreachable"})
|
||||
} else {
|
||||
nodeVersion := strings.TrimSpace(ping.Version)
|
||||
if nodeVersion == "" {
|
||||
nodeVersion = strings.TrimSpace(c.Agent.Version)
|
||||
}
|
||||
if expected != "" && nodeVersion != "" && nodeVersion != expected {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_version", Message: fmt.Sprintf("agent version mismatch: node=%s local=%s", nodeVersion, expected)})
|
||||
}
|
||||
if nodeVersion != "" {
|
||||
if ok, latest, known := s.CompareAgentVersion(nodeVersion); known && !ok {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "agent_outdated", Message: fmt.Sprintf("node version %s is older than latest known %s", nodeVersion, latest)})
|
||||
}
|
||||
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.AgentVersion) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.AgentVersion), nodeVersion) {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{
|
||||
Level: "warn",
|
||||
Kind: "baseline_agent_version",
|
||||
Message: fmt.Sprintf("baseline agent version mismatch: baseline=%s live=%s", dctrl.Baseline.AgentVersion, nodeVersion),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
softCtx, softCancel := context.WithTimeout(ctx, 7*time.Second)
|
||||
fresh, softErr := s.probeSoftware(softCtx, c, "", "")
|
||||
softCancel()
|
||||
if softErr != nil {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software_probe", Message: softErr.Error()})
|
||||
} else {
|
||||
oldSet := strings.TrimSpace(c.Software.Summary())
|
||||
newSet := strings.TrimSpace(fresh.Summary())
|
||||
if oldSet != "" && oldSet != "-" && newSet != oldSet {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{Level: "warn", Kind: "software", Message: fmt.Sprintf("software support changed: stored=%s live=%s", oldSet, newSet)})
|
||||
}
|
||||
if dctrl.Baseline.Enabled && strings.TrimSpace(dctrl.Baseline.Software) != "" && !strings.EqualFold(strings.TrimSpace(dctrl.Baseline.Software), newSet) {
|
||||
rep.Issues = append(rep.Issues, DriftIssue{
|
||||
Level: "warn",
|
||||
Kind: "baseline_software",
|
||||
Message: fmt.Sprintf("baseline software mismatch: baseline=%s live=%s", dctrl.Baseline.Software, newSet),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(rep.Issues, func(i, j int) bool { return rep.Issues[i].Kind < rep.Issues[j].Kind })
|
||||
return rep, nil
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/scrypt"
|
||||
)
|
||||
|
||||
const (
|
||||
exportMagic = "OBSCTLEXP1:"
|
||||
exportVersion = 2
|
||||
saltBytes = 16
|
||||
scryptN = 1 << 15
|
||||
scryptR = 8
|
||||
scryptP = 1
|
||||
)
|
||||
|
||||
// ExportBundle is the portable payload produced by Service.Export. It is
|
||||
// self-contained: it includes every cluster (with stored credentials),
|
||||
// telegram bot settings, locker config, and alert policies.
|
||||
type ExportBundle struct {
|
||||
Version int `json:"version"`
|
||||
ExportedAt string `json:"exported_at"`
|
||||
Registry Registry `json:"registry"`
|
||||
Files []ExportedFile `json:"files,omitempty"`
|
||||
}
|
||||
|
||||
type ExportedFile struct {
|
||||
OriginalPath string `json:"original_path"`
|
||||
Kind string `json:"kind"`
|
||||
Content []byte `json:"content"`
|
||||
}
|
||||
|
||||
// ImportMode controls how an imported bundle is merged into the current store.
|
||||
type ImportMode string
|
||||
|
||||
const (
|
||||
// ImportModeMerge adds clusters from the bundle; existing clusters with
|
||||
// the same name are replaced with the imported copy.
|
||||
ImportModeMerge ImportMode = "merge"
|
||||
// ImportModeReplace wipes the current registry and replaces it with the
|
||||
// imported bundle as-is.
|
||||
ImportModeReplace ImportMode = "replace"
|
||||
)
|
||||
|
||||
// ImportReport summarizes what happened during an import.
|
||||
type ImportReport struct {
|
||||
Added int
|
||||
Replaced int
|
||||
TotalAfter int
|
||||
TelegramApplied bool
|
||||
LockerApplied bool
|
||||
Mode ImportMode
|
||||
}
|
||||
|
||||
// Export writes an encrypted, passphrase-protected bundle of the full
|
||||
// registry to outPath. The bundle can be imported on another machine with
|
||||
// the same password — no master key transfer required.
|
||||
func (s *Service) Export(outPath, password string) error {
|
||||
if strings.TrimSpace(outPath) == "" {
|
||||
return errors.New("export: output path is required")
|
||||
}
|
||||
if strings.TrimSpace(password) == "" {
|
||||
return errors.New("export: password is required")
|
||||
}
|
||||
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: load registry: %w", err)
|
||||
}
|
||||
|
||||
files, err := collectExportFiles(reg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
bundle := ExportBundle{
|
||||
Version: exportVersion,
|
||||
ExportedAt: s.now().UTC().Format(time.RFC3339),
|
||||
Registry: reg,
|
||||
Files: files,
|
||||
}
|
||||
|
||||
payload, err := json.MarshalIndent(bundle, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: encode bundle: %w", err)
|
||||
}
|
||||
|
||||
salt := make([]byte, saltBytes)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return fmt.Errorf("export: generate salt: %w", err)
|
||||
}
|
||||
|
||||
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: derive key: %w", err)
|
||||
}
|
||||
|
||||
ciphertext, err := encrypt(payload, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("export: encrypt: %w", err)
|
||||
}
|
||||
|
||||
blob := append([]byte{}, salt...)
|
||||
blob = append(blob, ciphertext...)
|
||||
encoded := exportMagic + base64.StdEncoding.EncodeToString(blob) + "\n"
|
||||
|
||||
if dir := filepath.Dir(outPath); dir != "" && dir != "." {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return fmt.Errorf("export: create output dir: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
tmp := outPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(encoded), 0o600); err != nil {
|
||||
return fmt.Errorf("export: write temp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, outPath); err != nil {
|
||||
return fmt.Errorf("export: replace output: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Import reads an exported bundle from inPath using password, and applies
|
||||
// it to the local registry according to mode.
|
||||
func (s *Service) Import(inPath, password string, mode ImportMode) (ImportReport, error) {
|
||||
if strings.TrimSpace(inPath) == "" {
|
||||
return ImportReport{}, errors.New("import: input path is required")
|
||||
}
|
||||
if strings.TrimSpace(password) == "" {
|
||||
return ImportReport{}, errors.New("import: password is required")
|
||||
}
|
||||
if mode == "" {
|
||||
mode = ImportModeMerge
|
||||
}
|
||||
if mode != ImportModeMerge && mode != ImportModeReplace {
|
||||
return ImportReport{}, fmt.Errorf("import: unsupported mode %q", mode)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(inPath)
|
||||
if err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: read input: %w", err)
|
||||
}
|
||||
text := strings.TrimSpace(string(raw))
|
||||
if !strings.HasPrefix(text, exportMagic) {
|
||||
return ImportReport{}, errors.New("import: not a PXmon export bundle")
|
||||
}
|
||||
blob, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(text, exportMagic))
|
||||
if err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: decode payload: %w", err)
|
||||
}
|
||||
if len(blob) < saltBytes+aes.BlockSize {
|
||||
return ImportReport{}, errors.New("import: payload truncated")
|
||||
}
|
||||
salt := blob[:saltBytes]
|
||||
ciphertext := blob[saltBytes:]
|
||||
|
||||
key, err := scrypt.Key([]byte(password), salt, scryptN, scryptR, scryptP, masterKeyBytes)
|
||||
if err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: derive key: %w", err)
|
||||
}
|
||||
|
||||
plaintext, err := decrypt(ciphertext, key)
|
||||
if err != nil {
|
||||
return ImportReport{}, errors.New("import: wrong password or corrupt bundle")
|
||||
}
|
||||
|
||||
var bundle ExportBundle
|
||||
if err := json.Unmarshal(plaintext, &bundle); err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: parse bundle: %w", err)
|
||||
}
|
||||
if bundle.Version == 0 || bundle.Version > exportVersion {
|
||||
return ImportReport{}, fmt.Errorf("import: unsupported bundle version %d", bundle.Version)
|
||||
}
|
||||
|
||||
incoming := bundle.Registry
|
||||
if incoming.Clusters == nil {
|
||||
incoming.Clusters = []Cluster{}
|
||||
}
|
||||
if len(bundle.Files) > 0 {
|
||||
restored, restoreErr := s.restoreExportFiles(bundle.Files)
|
||||
if restoreErr != nil {
|
||||
return ImportReport{}, restoreErr
|
||||
}
|
||||
applyRestoredFilePaths(&incoming, restored)
|
||||
}
|
||||
|
||||
report := ImportReport{Mode: mode}
|
||||
|
||||
if mode == ImportModeReplace {
|
||||
report.Added = len(incoming.Clusters)
|
||||
report.TotalAfter = len(incoming.Clusters)
|
||||
report.TelegramApplied = incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0
|
||||
report.LockerApplied = incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled
|
||||
if err := s.store.Save(incoming); err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: save: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
current, err := s.store.Load()
|
||||
if err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: load current: %w", err)
|
||||
}
|
||||
if current.Clusters == nil {
|
||||
current.Clusters = []Cluster{}
|
||||
}
|
||||
|
||||
existingByName := make(map[string]int, len(current.Clusters))
|
||||
for i, c := range current.Clusters {
|
||||
existingByName[strings.ToLower(c.Name)] = i
|
||||
}
|
||||
|
||||
for _, inc := range incoming.Clusters {
|
||||
key := strings.ToLower(strings.TrimSpace(inc.Name))
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
inc.Alerts = ensureAlertPolicy(inc.Alerts)
|
||||
if idx, ok := existingByName[key]; ok {
|
||||
// Preserve original ID to keep references stable.
|
||||
inc.ID = current.Clusters[idx].ID
|
||||
current.Clusters[idx] = inc
|
||||
report.Replaced++
|
||||
} else {
|
||||
if strings.TrimSpace(inc.ID) == "" {
|
||||
inc.ID = newClusterID()
|
||||
}
|
||||
current.Clusters = append(current.Clusters, inc)
|
||||
existingByName[key] = len(current.Clusters) - 1
|
||||
report.Added++
|
||||
}
|
||||
}
|
||||
|
||||
if incoming.Telegram.Token != "" || incoming.Telegram.Enabled || len(incoming.Telegram.AllowedUserIDs) > 0 {
|
||||
current.Telegram = incoming.Telegram
|
||||
report.TelegramApplied = true
|
||||
}
|
||||
if incoming.Locker.PasswordHash != "" || incoming.Locker.Enabled {
|
||||
current.Locker = incoming.Locker
|
||||
report.LockerApplied = true
|
||||
}
|
||||
current.Backups = mergeBackupConfig(current.Backups, incoming.Backups)
|
||||
if strings.TrimSpace(current.ActiveClusterID) == "" {
|
||||
current.ActiveClusterID = incoming.ActiveClusterID
|
||||
}
|
||||
|
||||
report.TotalAfter = len(current.Clusters)
|
||||
if err := s.store.Save(current); err != nil {
|
||||
return ImportReport{}, fmt.Errorf("import: save: %w", err)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func collectExportFiles(reg Registry) ([]ExportedFile, error) {
|
||||
type wantFile struct {
|
||||
path string
|
||||
kind string
|
||||
}
|
||||
wants := make([]wantFile, 0)
|
||||
for _, c := range reg.Clusters {
|
||||
if strings.TrimSpace(c.KeyPath) != "" {
|
||||
wants = append(wants, wantFile{path: c.KeyPath, kind: "ssh_private_key"})
|
||||
}
|
||||
if strings.TrimSpace(c.KeyPassphraseFile) != "" {
|
||||
wants = append(wants, wantFile{path: c.KeyPassphraseFile, kind: "ssh_key_passphrase"})
|
||||
}
|
||||
}
|
||||
for _, t := range reg.Backups.Targets {
|
||||
if strings.TrimSpace(t.SFTPKeyPath) != "" {
|
||||
wants = append(wants, wantFile{path: t.SFTPKeyPath, kind: "sftp_private_key"})
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]struct{}{}
|
||||
files := make([]ExportedFile, 0, len(wants))
|
||||
for _, w := range wants {
|
||||
expanded, err := expandPath(w.path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("export: resolve %s file %q: %w", w.kind, w.path, err)
|
||||
}
|
||||
if _, ok := seen[expanded]; ok {
|
||||
continue
|
||||
}
|
||||
data, err := os.ReadFile(expanded)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("export: read %s file %q: %w", w.kind, expanded, err)
|
||||
}
|
||||
seen[expanded] = struct{}{}
|
||||
files = append(files, ExportedFile{
|
||||
OriginalPath: expanded,
|
||||
Kind: w.kind,
|
||||
Content: data,
|
||||
})
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (s *Service) restoreExportFiles(files []ExportedFile) (map[string]string, error) {
|
||||
base := filepath.Join(filepath.Dir(s.store.Path()), "imported-files")
|
||||
if err := os.MkdirAll(base, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("import: create imported-files dir: %w", err)
|
||||
}
|
||||
|
||||
restored := make(map[string]string, len(files))
|
||||
for _, f := range files {
|
||||
orig := strings.TrimSpace(f.OriginalPath)
|
||||
if orig == "" {
|
||||
continue
|
||||
}
|
||||
name := filepath.Base(orig)
|
||||
if name == "." || name == string(filepath.Separator) || strings.TrimSpace(name) == "" {
|
||||
name = "secret"
|
||||
}
|
||||
name = sanitizeExportFilename(name)
|
||||
sum := sha256.Sum256([]byte(orig))
|
||||
outPath := filepath.Join(base, hex.EncodeToString(sum[:6])+"-"+name)
|
||||
if err := os.WriteFile(outPath, f.Content, 0o600); err != nil {
|
||||
return nil, fmt.Errorf("import: restore file %q: %w", orig, err)
|
||||
}
|
||||
restored[orig] = outPath
|
||||
}
|
||||
return restored, nil
|
||||
}
|
||||
|
||||
func applyRestoredFilePaths(reg *Registry, restored map[string]string) {
|
||||
if reg == nil || len(restored) == 0 {
|
||||
return
|
||||
}
|
||||
lookup := func(path string) string {
|
||||
expanded, err := expandPath(path)
|
||||
if err == nil {
|
||||
if restoredPath := strings.TrimSpace(restored[expanded]); restoredPath != "" {
|
||||
return restoredPath
|
||||
}
|
||||
}
|
||||
if restoredPath := strings.TrimSpace(restored[strings.TrimSpace(path)]); restoredPath != "" {
|
||||
return restoredPath
|
||||
}
|
||||
return path
|
||||
}
|
||||
for i := range reg.Clusters {
|
||||
if strings.TrimSpace(reg.Clusters[i].KeyPath) != "" {
|
||||
reg.Clusters[i].KeyPath = lookup(reg.Clusters[i].KeyPath)
|
||||
}
|
||||
if strings.TrimSpace(reg.Clusters[i].KeyPassphraseFile) != "" {
|
||||
reg.Clusters[i].KeyPassphraseFile = lookup(reg.Clusters[i].KeyPassphraseFile)
|
||||
}
|
||||
}
|
||||
for i := range reg.Backups.Targets {
|
||||
if strings.TrimSpace(reg.Backups.Targets[i].SFTPKeyPath) != "" {
|
||||
reg.Backups.Targets[i].SFTPKeyPath = lookup(reg.Backups.Targets[i].SFTPKeyPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeExportFilename(name string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range name {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
b.WriteRune(r)
|
||||
case r >= 'A' && r <= 'Z':
|
||||
b.WriteRune(r)
|
||||
case r >= '0' && r <= '9':
|
||||
b.WriteRune(r)
|
||||
case r == '.', r == '_', r == '-':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
out := strings.Trim(b.String(), ".")
|
||||
if out == "" {
|
||||
return "secret"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeBackupConfig(current, incoming BackupConfig) BackupConfig {
|
||||
if len(incoming.Targets) == 0 && len(incoming.Plans) == 0 {
|
||||
return current
|
||||
}
|
||||
if current.Targets == nil {
|
||||
current.Targets = []BackupTarget{}
|
||||
}
|
||||
if current.Plans == nil {
|
||||
current.Plans = []BackupPlan{}
|
||||
}
|
||||
|
||||
targetIDMap := map[string]string{}
|
||||
targetByName := map[string]int{}
|
||||
for i, t := range current.Targets {
|
||||
if key := strings.ToLower(strings.TrimSpace(t.Name)); key != "" {
|
||||
targetByName[key] = i
|
||||
}
|
||||
}
|
||||
for _, inc := range incoming.Targets {
|
||||
if strings.TrimSpace(inc.ID) == "" {
|
||||
inc.ID = newClusterID()
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSpace(inc.Name))
|
||||
if key != "" {
|
||||
if idx, ok := targetByName[key]; ok {
|
||||
oldID := current.Targets[idx].ID
|
||||
targetIDMap[inc.ID] = oldID
|
||||
inc.ID = oldID
|
||||
current.Targets[idx] = inc
|
||||
continue
|
||||
}
|
||||
}
|
||||
current.Targets = append(current.Targets, inc)
|
||||
if key != "" {
|
||||
targetByName[key] = len(current.Targets) - 1
|
||||
}
|
||||
}
|
||||
|
||||
planByName := map[string]int{}
|
||||
for i, p := range current.Plans {
|
||||
if key := strings.ToLower(strings.TrimSpace(p.Name)); key != "" {
|
||||
planByName[key] = i
|
||||
}
|
||||
}
|
||||
for _, inc := range incoming.Plans {
|
||||
if mapped := strings.TrimSpace(targetIDMap[inc.TargetID]); mapped != "" {
|
||||
inc.TargetID = mapped
|
||||
}
|
||||
if strings.TrimSpace(inc.ID) == "" {
|
||||
inc.ID = newClusterID()
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSpace(inc.Name))
|
||||
if key != "" {
|
||||
if idx, ok := planByName[key]; ok {
|
||||
inc.ID = current.Plans[idx].ID
|
||||
current.Plans[idx] = inc
|
||||
continue
|
||||
}
|
||||
}
|
||||
current.Plans = append(current.Plans, inc)
|
||||
if key != "" {
|
||||
planByName[key] = len(current.Plans) - 1
|
||||
}
|
||||
}
|
||||
return current
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const currentVersion = 8
|
||||
|
||||
// AuthMethod describes how SSH authentication is performed.
|
||||
type AuthMethod string
|
||||
|
||||
const (
|
||||
AuthMethodPassword AuthMethod = "password"
|
||||
AuthMethodKey AuthMethod = "key"
|
||||
)
|
||||
|
||||
// TransportMode selects how pxmon reaches the agent HTTP API on a node.
|
||||
type TransportMode string
|
||||
|
||||
const (
|
||||
// TransportDirect is the default: pxmon dials the agent's listen
|
||||
// address over plain TCP from the local machine.
|
||||
TransportDirect TransportMode = "direct"
|
||||
// TransportIPFabric tunnels the agent HTTP call through the existing SSH
|
||||
// connection. Meant for nodes on ipfabric-style networking where the node
|
||||
// has no default outbound route and we must not touch its network config.
|
||||
// The agent is expected to bind to 127.0.0.1 on the node.
|
||||
TransportIPFabric TransportMode = "ipfabric"
|
||||
)
|
||||
|
||||
func normalizeTransport(t TransportMode) TransportMode {
|
||||
switch strings.ToLower(strings.TrimSpace(string(t))) {
|
||||
case "ipfabric", "ip-fabric", "ip_fabric":
|
||||
return TransportIPFabric
|
||||
case "", "direct":
|
||||
return TransportDirect
|
||||
default:
|
||||
return TransportDirect
|
||||
}
|
||||
}
|
||||
|
||||
// Registry is the local inventory of managed clusters (nodes).
|
||||
type Registry struct {
|
||||
Version int `json:"version"`
|
||||
ActiveClusterID string `json:"active_cluster_id,omitempty"`
|
||||
Telegram Telegram `json:"telegram,omitempty"`
|
||||
Locker Locker `json:"locker,omitempty"`
|
||||
Backups BackupConfig `json:"backups,omitempty"`
|
||||
Clusters []Cluster `json:"clusters"`
|
||||
}
|
||||
|
||||
type BackupConfig struct {
|
||||
Targets []BackupTarget `json:"targets,omitempty"`
|
||||
Plans []BackupPlan `json:"plans,omitempty"`
|
||||
}
|
||||
|
||||
type BackupTarget struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"` // sftp|s3
|
||||
Enabled bool `json:"enabled"`
|
||||
|
||||
SFTPHost string `json:"sftp_host,omitempty"`
|
||||
SFTPPort int `json:"sftp_port,omitempty"`
|
||||
SFTPUser string `json:"sftp_user,omitempty"`
|
||||
SFTPPassword string `json:"sftp_password,omitempty"`
|
||||
SFTPKeyPath string `json:"sftp_key_path,omitempty"`
|
||||
SFTPBasePath string `json:"sftp_base_path,omitempty"`
|
||||
|
||||
S3Endpoint string `json:"s3_endpoint,omitempty"`
|
||||
S3Region string `json:"s3_region,omitempty"`
|
||||
S3Bucket string `json:"s3_bucket,omitempty"`
|
||||
S3Prefix string `json:"s3_prefix,omitempty"`
|
||||
S3AccessKey string `json:"s3_access_key,omitempty"`
|
||||
S3SecretKey string `json:"s3_secret_key,omitempty"`
|
||||
S3UseSSL bool `json:"s3_use_ssl,omitempty"`
|
||||
S3PathStyle bool `json:"s3_path_style,omitempty"`
|
||||
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type BackupPlan struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
TargetID string `json:"target_id"`
|
||||
Paths []string `json:"paths"`
|
||||
Every string `json:"every,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
RetainDays int `json:"retain_days,omitempty"`
|
||||
Compress bool `json:"compress"`
|
||||
LastRunAt time.Time `json:"last_run_at,omitempty"`
|
||||
LastStatus string `json:"last_status,omitempty"`
|
||||
LastArchive string `json:"last_archive,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// Telegram stores Telegram bot integration settings.
|
||||
type Telegram struct {
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
Token string `json:"token,omitempty"`
|
||||
AllowedUserIDs []int64 `json:"allowed_user_ids,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
// Locker stores global UI/CLI lock settings.
|
||||
type Locker struct {
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
PasswordHash string `json:"password_hash,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
// Cluster describes one managed node that we can reach over SSH.
|
||||
type Cluster struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
Transport TransportMode `json:"transport,omitempty"`
|
||||
AuthMethod AuthMethod `json:"auth_method"`
|
||||
Password string `json:"password,omitempty"`
|
||||
KeyPath string `json:"key_path,omitempty"`
|
||||
KeyPassphrase string `json:"key_passphrase,omitempty"`
|
||||
KeyPassphraseFile string `json:"key_passphrase_file,omitempty"`
|
||||
InsecureHostKey bool `json:"insecure_host_key,omitempty"`
|
||||
Alerts AlertPolicy `json:"alerts"`
|
||||
VMAlerts VMAlertPolicy `json:"vm_alerts,omitempty"`
|
||||
AlertRouting AlertRoutingPolicy `json:"alert_routing,omitempty"`
|
||||
RepoTunnel RepoTunnelState `json:"repo_tunnel,omitempty"`
|
||||
RunbookTrigger RunbookTrigger `json:"runbook_trigger,omitempty"`
|
||||
Drift DriftControl `json:"drift,omitempty"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
KVMTags map[string][]string `json:"kvm_tags,omitempty"`
|
||||
Agent AgentInstall `json:"agent,omitempty"`
|
||||
Software SoftwareInfo `json:"software,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// AlertPolicy defines warning thresholds used by CLI monitor mode.
|
||||
type AlertPolicy struct {
|
||||
CPUWarnPercent float64 `json:"cpu_warn_percent"`
|
||||
RAMWarnPercent float64 `json:"ram_warn_percent"`
|
||||
SwapWarnPercent float64 `json:"swap_warn_percent"`
|
||||
DiskWarnPercent float64 `json:"disk_warn_percent"`
|
||||
NetWarnMbps float64 `json:"net_warn_mbps"`
|
||||
NetSustainEnabled bool `json:"net_sustain_enabled,omitempty"`
|
||||
NetSustainIface string `json:"net_sustain_iface,omitempty"`
|
||||
NetSustainInclude []string `json:"net_sustain_include,omitempty"`
|
||||
NetSustainExclude []string `json:"net_sustain_exclude,omitempty"`
|
||||
NetSustainMbps float64 `json:"net_sustain_mbps,omitempty"`
|
||||
NetSustainMinutes int `json:"net_sustain_minutes,omitempty"`
|
||||
NetSustainCooldownMins int `json:"net_sustain_cooldown_mins,omitempty"`
|
||||
}
|
||||
|
||||
// VMAlertPolicy controls KVM VM-state alerting per cluster.
|
||||
type VMAlertPolicy struct {
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
WarnOnShutoff bool `json:"warn_on_shutoff,omitempty"`
|
||||
MinRunning int `json:"min_running,omitempty"`
|
||||
}
|
||||
|
||||
// AlertRoutingPolicy controls delivery behavior for alerts (e.g. Telegram).
|
||||
type AlertRoutingPolicy struct {
|
||||
CriticalImmediate bool `json:"critical_immediate,omitempty"`
|
||||
WarningBatchMins int `json:"warning_batch_mins,omitempty"`
|
||||
}
|
||||
|
||||
// RunbookTrigger controls automatic runbook execution on selected events.
|
||||
type RunbookTrigger struct {
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
OnVMShutoff bool `json:"on_vm_shutoff,omitempty"`
|
||||
RunbookID string `json:"runbook_id,omitempty"`
|
||||
CooldownMins int `json:"cooldown_mins,omitempty"`
|
||||
LastTriggered time.Time `json:"last_triggered,omitempty"`
|
||||
}
|
||||
|
||||
// DriftBaseline stores expected values for drift comparisons.
|
||||
type DriftBaseline struct {
|
||||
Enabled bool `json:"enabled,omitempty"`
|
||||
SetAt time.Time `json:"set_at,omitempty"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
Software string `json:"software,omitempty"`
|
||||
}
|
||||
|
||||
// DriftControl stores baseline and per-issue acknowledgement windows.
|
||||
type DriftControl struct {
|
||||
Baseline DriftBaseline `json:"baseline,omitempty"`
|
||||
AckUntil map[string]time.Time `json:"ack_until,omitempty"`
|
||||
}
|
||||
|
||||
// AgentInstall describes remote pxmon-agent installation details.
|
||||
type AgentInstall struct {
|
||||
Installed bool `json:"installed"`
|
||||
Version string `json:"version,omitempty"`
|
||||
RemoteBinary string `json:"remote_binary,omitempty"`
|
||||
RemoteConfig string `json:"remote_config,omitempty"`
|
||||
RemoteLog string `json:"remote_log,omitempty"`
|
||||
RemotePIDFile string `json:"remote_pid_file,omitempty"`
|
||||
ListenAddress string `json:"listen_address,omitempty"`
|
||||
Port int `json:"port,omitempty"`
|
||||
Token string `json:"token,omitempty"`
|
||||
RequestSecret string `json:"request_secret,omitempty"`
|
||||
TLSEnabled bool `json:"tls_enabled,omitempty"`
|
||||
TLSCertPath string `json:"tls_cert_path,omitempty"`
|
||||
TLSKeyPath string `json:"tls_key_path,omitempty"`
|
||||
TLSFingerprint string `json:"tls_fingerprint,omitempty"`
|
||||
LastBootstrapAt time.Time `json:"last_bootstrap_at,omitempty"`
|
||||
}
|
||||
|
||||
// SoftwareInfo describes discovered software/plugins on a node.
|
||||
type SoftwareInfo struct {
|
||||
DetectedAt time.Time `json:"detected_at,omitempty"`
|
||||
Bird bool `json:"bird,omitempty"`
|
||||
FRR bool `json:"frr,omitempty"`
|
||||
KVM bool `json:"kvm,omitempty"`
|
||||
LXC bool `json:"lxc,omitempty"`
|
||||
LXD bool `json:"lxd,omitempty"`
|
||||
Versions map[string]string `json:"versions,omitempty"`
|
||||
}
|
||||
|
||||
func (s SoftwareInfo) SupportedList() []string {
|
||||
out := make([]string, 0, 5)
|
||||
if s.Bird {
|
||||
out = append(out, "bird")
|
||||
}
|
||||
if s.FRR {
|
||||
out = append(out, "frr")
|
||||
}
|
||||
if s.KVM {
|
||||
out = append(out, "kvm")
|
||||
}
|
||||
if s.LXC {
|
||||
out = append(out, "lxc")
|
||||
}
|
||||
if s.LXD {
|
||||
out = append(out, "lxd")
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (s SoftwareInfo) Summary() string {
|
||||
list := s.SupportedList()
|
||||
if len(list) == 0 {
|
||||
if !s.DetectedAt.IsZero() {
|
||||
return "none"
|
||||
}
|
||||
return "-"
|
||||
}
|
||||
return strings.Join(list, ",")
|
||||
}
|
||||
|
||||
func newRegistry() Registry {
|
||||
return Registry{
|
||||
Version: currentVersion,
|
||||
Backups: normalizeBackupConfig(BackupConfig{}),
|
||||
Clusters: []Cluster{},
|
||||
}
|
||||
}
|
||||
|
||||
func defaultAlertPolicy() AlertPolicy {
|
||||
return AlertPolicy{
|
||||
CPUWarnPercent: 85,
|
||||
RAMWarnPercent: 90,
|
||||
SwapWarnPercent: 80,
|
||||
DiskWarnPercent: 90,
|
||||
NetWarnMbps: 300,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultVMAlertPolicy() VMAlertPolicy {
|
||||
return VMAlertPolicy{
|
||||
Enabled: false,
|
||||
WarnOnShutoff: true,
|
||||
MinRunning: 1,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultAlertRoutingPolicy() AlertRoutingPolicy {
|
||||
return AlertRoutingPolicy{
|
||||
CriticalImmediate: true,
|
||||
WarningBatchMins: 5,
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRunbookTrigger() RunbookTrigger {
|
||||
return RunbookTrigger{
|
||||
Enabled: false,
|
||||
OnVMShutoff: true,
|
||||
CooldownMins: 30,
|
||||
}
|
||||
}
|
||||
|
||||
func ensureAlertPolicy(p AlertPolicy) AlertPolicy {
|
||||
d := defaultAlertPolicy()
|
||||
if p.CPUWarnPercent <= 0 {
|
||||
p.CPUWarnPercent = d.CPUWarnPercent
|
||||
}
|
||||
if p.RAMWarnPercent <= 0 {
|
||||
p.RAMWarnPercent = d.RAMWarnPercent
|
||||
}
|
||||
if p.SwapWarnPercent <= 0 {
|
||||
p.SwapWarnPercent = d.SwapWarnPercent
|
||||
}
|
||||
if p.DiskWarnPercent <= 0 {
|
||||
p.DiskWarnPercent = d.DiskWarnPercent
|
||||
}
|
||||
if p.NetWarnMbps <= 0 {
|
||||
p.NetWarnMbps = d.NetWarnMbps
|
||||
}
|
||||
if p.NetSustainEnabled {
|
||||
if p.NetSustainMbps <= 0 {
|
||||
p.NetSustainMbps = d.NetWarnMbps
|
||||
}
|
||||
if p.NetSustainMinutes <= 0 {
|
||||
p.NetSustainMinutes = 60
|
||||
}
|
||||
if p.NetSustainCooldownMins <= 0 {
|
||||
p.NetSustainCooldownMins = 30
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func ensureVMAlertPolicy(p VMAlertPolicy) VMAlertPolicy {
|
||||
d := defaultVMAlertPolicy()
|
||||
wasZero := p == (VMAlertPolicy{})
|
||||
if p.MinRunning <= 0 {
|
||||
p.MinRunning = d.MinRunning
|
||||
}
|
||||
if !p.WarnOnShutoff {
|
||||
// Keep explicit false if user set it, but default to true for zero-value
|
||||
// policy loaded from old configs.
|
||||
if wasZero {
|
||||
p.WarnOnShutoff = d.WarnOnShutoff
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func ensureAlertRoutingPolicy(p AlertRoutingPolicy) AlertRoutingPolicy {
|
||||
d := defaultAlertRoutingPolicy()
|
||||
if p.WarningBatchMins <= 0 {
|
||||
p.WarningBatchMins = d.WarningBatchMins
|
||||
}
|
||||
// default true when unset
|
||||
if !p.CriticalImmediate && p == (AlertRoutingPolicy{}) {
|
||||
p.CriticalImmediate = d.CriticalImmediate
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func ensureRunbookTrigger(p RunbookTrigger) RunbookTrigger {
|
||||
d := defaultRunbookTrigger()
|
||||
if p.CooldownMins <= 0 {
|
||||
p.CooldownMins = d.CooldownMins
|
||||
}
|
||||
if !p.OnVMShutoff && p == (RunbookTrigger{}) {
|
||||
p.OnVMShutoff = d.OnVMShutoff
|
||||
}
|
||||
p.RunbookID = strings.TrimSpace(p.RunbookID)
|
||||
return p
|
||||
}
|
||||
|
||||
func normalizeBackupConfig(cfg BackupConfig) BackupConfig {
|
||||
if cfg.Targets == nil {
|
||||
cfg.Targets = []BackupTarget{}
|
||||
}
|
||||
if cfg.Plans == nil {
|
||||
cfg.Plans = []BackupPlan{}
|
||||
}
|
||||
for i := range cfg.Targets {
|
||||
t := &cfg.Targets[i]
|
||||
t.ID = strings.TrimSpace(t.ID)
|
||||
t.Name = strings.TrimSpace(t.Name)
|
||||
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
|
||||
if t.SFTPPort <= 0 {
|
||||
t.SFTPPort = 22
|
||||
}
|
||||
if t.ID == "" {
|
||||
t.ID = newClusterID()
|
||||
}
|
||||
if t.Name == "" {
|
||||
t.Name = t.ID
|
||||
}
|
||||
if t.Type != "sftp" && t.Type != "s3" {
|
||||
t.Type = "sftp"
|
||||
}
|
||||
if !t.Enabled && t.CreatedAt.IsZero() {
|
||||
t.Enabled = true
|
||||
}
|
||||
if t.CreatedAt.IsZero() {
|
||||
t.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
if t.UpdatedAt.IsZero() {
|
||||
t.UpdatedAt = t.CreatedAt
|
||||
}
|
||||
}
|
||||
for i := range cfg.Plans {
|
||||
p := &cfg.Plans[i]
|
||||
p.ID = strings.TrimSpace(p.ID)
|
||||
p.Name = strings.TrimSpace(p.Name)
|
||||
p.Cluster = strings.TrimSpace(p.Cluster)
|
||||
p.TargetID = strings.TrimSpace(p.TargetID)
|
||||
if p.ID == "" {
|
||||
p.ID = newClusterID()
|
||||
}
|
||||
if p.Name == "" {
|
||||
p.Name = p.ID
|
||||
}
|
||||
if p.Paths == nil {
|
||||
p.Paths = []string{}
|
||||
}
|
||||
if p.Every == "" {
|
||||
p.Every = "24h"
|
||||
}
|
||||
if p.RetainDays <= 0 {
|
||||
p.RetainDays = 30
|
||||
}
|
||||
if !p.Compress {
|
||||
p.Compress = true
|
||||
}
|
||||
if !p.Enabled && p.CreatedAt.IsZero() {
|
||||
p.Enabled = true
|
||||
}
|
||||
if p.CreatedAt.IsZero() {
|
||||
p.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
if p.UpdatedAt.IsZero() {
|
||||
p.UpdatedAt = p.CreatedAt
|
||||
}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
func normalizeDriftControl(d DriftControl) DriftControl {
|
||||
if len(d.AckUntil) == 0 {
|
||||
d.AckUntil = nil
|
||||
return d
|
||||
}
|
||||
out := make(map[string]time.Time, len(d.AckUntil))
|
||||
for k, v := range d.AckUntil {
|
||||
n := strings.ToLower(strings.TrimSpace(k))
|
||||
if n == "" || v.IsZero() {
|
||||
continue
|
||||
}
|
||||
out[n] = v.UTC()
|
||||
}
|
||||
if len(out) == 0 {
|
||||
d.AckUntil = nil
|
||||
} else {
|
||||
d.AckUntil = out
|
||||
}
|
||||
d.Baseline.AgentVersion = strings.TrimSpace(d.Baseline.AgentVersion)
|
||||
d.Baseline.Software = strings.TrimSpace(d.Baseline.Software)
|
||||
return d
|
||||
}
|
||||
|
||||
func normalizeTagList(tags []string) []string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(tags))
|
||||
out := make([]string, 0, len(tags))
|
||||
for _, t := range tags {
|
||||
n := strings.ToLower(strings.TrimSpace(t))
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[n]; ok {
|
||||
continue
|
||||
}
|
||||
seen[n] = struct{}{}
|
||||
out = append(out, n)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeVMTagMap(m map[string][]string) map[string][]string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string][]string, len(m))
|
||||
for vm, tags := range m {
|
||||
vmName := strings.TrimSpace(vm)
|
||||
if vmName == "" {
|
||||
continue
|
||||
}
|
||||
norm := normalizeTagList(tags)
|
||||
if len(norm) == 0 {
|
||||
continue
|
||||
}
|
||||
out[vmName] = norm
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func normalizeLocker(cfg Locker) Locker {
|
||||
cfg.PasswordHash = strings.TrimSpace(cfg.PasswordHash)
|
||||
if cfg.PasswordHash == "" {
|
||||
cfg.Enabled = false
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
|
||||
"pxmon/internal/history"
|
||||
)
|
||||
|
||||
type InterfaceP95Snapshot struct {
|
||||
ClusterName string `json:"cluster"`
|
||||
ClusterID string `json:"cluster_id"`
|
||||
Interface string `json:"interface"`
|
||||
Range history.RangeShortcut `json:"range"`
|
||||
Samples int `json:"samples"`
|
||||
P95Mbps float64 `json:"p95_mbps"`
|
||||
AvgMbps float64 `json:"avg_mbps"`
|
||||
MaxMbps float64 `json:"max_mbps"`
|
||||
Series []history.NodeSamplePoint `json:"series,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) CollectInterfaceP95(selector, iface string, rng history.RangeShortcut) (InterfaceP95Snapshot, error) {
|
||||
iface = strings.TrimSpace(iface)
|
||||
if iface == "" {
|
||||
return InterfaceP95Snapshot{}, errors.New("interface is required")
|
||||
}
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return InterfaceP95Snapshot{}, err
|
||||
}
|
||||
store := s.NetworkStore()
|
||||
if store == nil {
|
||||
return InterfaceP95Snapshot{}, errors.New("history store not configured")
|
||||
}
|
||||
snaps, err := store.Load(c.ID, rng.Since(s.now()))
|
||||
if err != nil {
|
||||
return InterfaceP95Snapshot{}, err
|
||||
}
|
||||
series := history.AggregateNodeSeries(snaps, iface)
|
||||
out := InterfaceP95Snapshot{
|
||||
ClusterName: c.Name,
|
||||
ClusterID: c.ID,
|
||||
Interface: iface,
|
||||
Range: rng,
|
||||
Samples: len(series),
|
||||
P95Mbps: history.PercentileMbps(series, 95),
|
||||
Series: series,
|
||||
}
|
||||
if len(series) == 0 {
|
||||
return out, nil
|
||||
}
|
||||
var sum, maxV float64
|
||||
for _, p := range series {
|
||||
sum += p.TotalMbps
|
||||
if p.TotalMbps > maxV {
|
||||
maxV = p.TotalMbps
|
||||
}
|
||||
}
|
||||
out.AvgMbps = sum / float64(len(series))
|
||||
out.MaxMbps = maxV
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) RenderInterfaceP95GraphPNG(snap InterfaceP95Snapshot) ([]byte, error) {
|
||||
if len(snap.Series) == 0 {
|
||||
return history.RenderNodeNetworkPNG([]history.NodeSamplePoint{}, history.ChartOptions{
|
||||
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
|
||||
Subtitle: "No samples",
|
||||
})
|
||||
}
|
||||
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps", snap.P95Mbps, snap.MaxMbps, snap.AvgMbps)
|
||||
if math.IsNaN(snap.P95Mbps) {
|
||||
subtitle = "No samples"
|
||||
}
|
||||
return history.RenderNodeNetworkPNG(snap.Series, history.ChartOptions{
|
||||
Title: fmt.Sprintf("%s: %s", snap.ClusterName, snap.Interface),
|
||||
Subtitle: subtitle,
|
||||
Percentile: 95,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (s *Service) GetAlertRouting(selector string) (AlertRoutingPolicy, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return AlertRoutingPolicy{}, err
|
||||
}
|
||||
return ensureAlertRoutingPolicy(c.AlertRouting), nil
|
||||
}
|
||||
|
||||
func (s *Service) SetAlertRouting(selector string, p AlertRoutingPolicy) (Cluster, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c.AlertRouting = ensureAlertRoutingPolicy(p)
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("alert.routing", c.Name, "updated")
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetRunbookTrigger(selector string) (RunbookTrigger, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return RunbookTrigger{}, err
|
||||
}
|
||||
return ensureRunbookTrigger(c.RunbookTrigger), nil
|
||||
}
|
||||
|
||||
func (s *Service) SetRunbookTrigger(selector string, p RunbookTrigger) (Cluster, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c.RunbookTrigger = ensureRunbookTrigger(p)
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("runbook.trigger", c.Name, "updated")
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) TouchRunbookTrigger(selector string, when time.Time) error {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tr := ensureRunbookTrigger(c.RunbookTrigger)
|
||||
tr.LastTriggered = when.UTC()
|
||||
c.RunbookTrigger = tr
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
return s.store.Save(reg)
|
||||
}
|
||||
|
||||
func (s *Service) SetDriftBaseline(selector string) (Cluster, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
d := normalizeDriftControl(c.Drift)
|
||||
d.Baseline = DriftBaseline{
|
||||
Enabled: true,
|
||||
SetAt: s.now().UTC(),
|
||||
AgentVersion: strings.TrimSpace(c.Agent.Version),
|
||||
Software: strings.TrimSpace(c.Software.Summary()),
|
||||
}
|
||||
c.Drift = d
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("drift.baseline", c.Name, "set")
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetDriftControl(selector string) (DriftControl, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return DriftControl{}, err
|
||||
}
|
||||
return normalizeDriftControl(c.Drift), nil
|
||||
}
|
||||
|
||||
func (s *Service) AckDriftIssue(selector, issueKind string, until time.Time) (Cluster, error) {
|
||||
kind := strings.ToLower(strings.TrimSpace(issueKind))
|
||||
if kind == "" {
|
||||
return Cluster{}, errors.New("issue kind is required")
|
||||
}
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
d := normalizeDriftControl(c.Drift)
|
||||
if d.AckUntil == nil {
|
||||
d.AckUntil = map[string]time.Time{}
|
||||
}
|
||||
d.AckUntil[kind] = until.UTC()
|
||||
c.Drift = d
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("drift.ack", c.Name, kind+" until="+until.UTC().Format(time.RFC3339))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) IsDriftIssueAcked(c Cluster, kind string, now time.Time) bool {
|
||||
d := normalizeDriftControl(c.Drift)
|
||||
if len(d.AckUntil) == 0 {
|
||||
return false
|
||||
}
|
||||
u, ok := d.AckUntil[strings.ToLower(strings.TrimSpace(kind))]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
return now.UTC().Before(u)
|
||||
}
|
||||
@@ -0,0 +1,413 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type RepoTunnelOptions struct {
|
||||
Gateway string
|
||||
GatewayIP string
|
||||
Table int
|
||||
Priority int
|
||||
PackageManager string
|
||||
Command string
|
||||
KeepEnabled bool
|
||||
NoRule bool
|
||||
}
|
||||
|
||||
type RepoTunnelState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Proxy string `json:"proxy,omitempty"`
|
||||
Source string `json:"source,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) RepoTunnelEnable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
|
||||
script, err := repoTunnelEnableScript(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := s.RunRemoteShell(ctx, selector, script)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
gw, err := parseRepoTunnelGateway(opts.Gateway)
|
||||
if err == nil {
|
||||
_ = s.updateRepoTunnelState(selector, RepoTunnelState{
|
||||
Enabled: true,
|
||||
Proxy: gw.proxyURL,
|
||||
Source: strings.ToLower(strings.TrimSpace(opts.PackageManager)),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) RepoTunnelDisable(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
|
||||
script, err := repoTunnelDisableScript(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := s.RunRemoteShell(ctx, selector, script)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = s.updateRepoTunnelState(selector, RepoTunnelState{})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) RepoTunnelState(ctx context.Context, selector string) (RepoTunnelState, error) {
|
||||
out, err := s.RunRemoteShell(ctx, selector, repoTunnelDetectScript())
|
||||
if err != nil {
|
||||
return RepoTunnelState{}, err
|
||||
}
|
||||
state := RepoTunnelState{}
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case strings.HasPrefix(line, "proxy="):
|
||||
state.Proxy = strings.TrimSpace(strings.TrimPrefix(line, "proxy="))
|
||||
case strings.HasPrefix(line, "source="):
|
||||
state.Source = strings.TrimSpace(strings.TrimPrefix(line, "source="))
|
||||
}
|
||||
}
|
||||
state.Enabled = state.Proxy != ""
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func (s *Service) updateRepoTunnelState(selector string, state RepoTunnelState) error {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
c.RepoTunnel = state
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
return s.store.Save(reg)
|
||||
}
|
||||
|
||||
func (s *Service) RepoTunnelStatus(ctx context.Context, selector string) (string, error) {
|
||||
return s.RunRemoteShell(ctx, selector, repoTunnelStatusScript())
|
||||
}
|
||||
|
||||
func (s *Service) RepoTunnelInstall(ctx context.Context, selector string, opts RepoTunnelOptions) (string, error) {
|
||||
if strings.TrimSpace(opts.Command) == "" {
|
||||
return "", errors.New("install command is required")
|
||||
}
|
||||
enableScript, err := repoTunnelEnableScript(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
disableScript, err := repoTunnelDisableScript(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
body := enableScript + "\n" + strings.TrimSpace(opts.Command) + "\n"
|
||||
if !opts.KeepEnabled {
|
||||
body = enableScript + "\ncleanup_pxmon_repo_tunnel() {\n" + disableScript + "\n}\ntrap cleanup_pxmon_repo_tunnel EXIT\n" + strings.TrimSpace(opts.Command) + "\n"
|
||||
}
|
||||
return s.RunRemoteShell(ctx, selector, body)
|
||||
}
|
||||
|
||||
func RepoTunnelGatewayScript(port int, allowCIDRs []string) (string, error) {
|
||||
if port == 0 {
|
||||
port = 3128
|
||||
}
|
||||
if port < 1 || port > 65535 {
|
||||
return "", errors.New("--port must be in range 1..65535")
|
||||
}
|
||||
if len(allowCIDRs) == 0 {
|
||||
return "", errors.New("at least one --allow CIDR/IP is required")
|
||||
}
|
||||
aclParts := make([]string, 0, len(allowCIDRs))
|
||||
for _, raw := range allowCIDRs {
|
||||
v := strings.TrimSpace(raw)
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
if !validSquidSrcACL(v) {
|
||||
return "", fmt.Errorf("invalid --allow %q; use an IP or CIDR without spaces", raw)
|
||||
}
|
||||
aclParts = append(aclParts, v)
|
||||
}
|
||||
if len(aclParts) == 0 {
|
||||
return "", errors.New("at least one --allow CIDR/IP is required")
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`set -eu
|
||||
if command -v dnf >/dev/null 2>&1; then
|
||||
dnf install -y squid
|
||||
elif command -v yum >/dev/null 2>&1; then
|
||||
yum install -y squid
|
||||
elif command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y squid
|
||||
else
|
||||
echo "no supported package manager found for squid install" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
conf=/etc/squid/squid.conf
|
||||
cp -a "$conf" "$conf.pxmon-bak.$(date +%%Y%%m%%d%%H%%M%%S)"
|
||||
awk '
|
||||
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
|
||||
/# END PXMON REPO TUNNEL/ {skip=0; next}
|
||||
!skip {print}
|
||||
' "$conf" > "$conf.tmp"
|
||||
mv "$conf.tmp" "$conf"
|
||||
|
||||
block=$(mktemp)
|
||||
{
|
||||
echo "# BEGIN PXMON REPO TUNNEL"
|
||||
if ! grep -Eq "^http_port[[:space:]]+([^[:space:]]+:)?%d\b" "$conf"; then
|
||||
echo "http_port %d"
|
||||
fi
|
||||
echo "acl pxmon_repo_tunnel src %s"
|
||||
echo "http_access allow pxmon_repo_tunnel"
|
||||
echo "# END PXMON REPO TUNNEL"
|
||||
} > "$block"
|
||||
|
||||
if grep -q "^http_access deny all" "$conf"; then
|
||||
awk -v block="$block" '
|
||||
BEGIN {while ((getline line < block) > 0) b = b line "\n"; close(block); inserted=0}
|
||||
/^http_access deny all/ && !inserted {printf "%%s", b; inserted=1}
|
||||
{print}
|
||||
END {if (!inserted) printf "%%s", b}
|
||||
' "$conf" > "$conf.tmp"
|
||||
mv "$conf.tmp" "$conf"
|
||||
else
|
||||
cat "$block" >> "$conf"
|
||||
fi
|
||||
rm -f "$block"
|
||||
|
||||
systemctl enable --now squid
|
||||
systemctl restart squid
|
||||
if command -v firewall-cmd >/dev/null 2>&1 && systemctl is-active --quiet firewalld; then
|
||||
firewall-cmd --add-port=%d/tcp --permanent
|
||||
firewall-cmd --reload
|
||||
fi
|
||||
echo "pxmon repo gateway ready on port %d"
|
||||
`, port, port, strings.Join(aclParts, " "), port, port), nil
|
||||
}
|
||||
|
||||
func validSquidSrcACL(v string) bool {
|
||||
for _, r := range v {
|
||||
if r >= 'a' && r <= 'z' {
|
||||
continue
|
||||
}
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
continue
|
||||
}
|
||||
if r >= '0' && r <= '9' {
|
||||
continue
|
||||
}
|
||||
switch r {
|
||||
case '.', ':', '/', '_', '-':
|
||||
continue
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return v != ""
|
||||
}
|
||||
|
||||
func repoTunnelEnableScript(opts RepoTunnelOptions) (string, error) {
|
||||
gw, err := parseRepoTunnelGateway(opts.Gateway)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !opts.NoRule && opts.Table <= 0 {
|
||||
return "", errors.New("--table is required unless --no-rule is used")
|
||||
}
|
||||
manager := strings.ToLower(strings.TrimSpace(opts.PackageManager))
|
||||
if manager == "" {
|
||||
manager = "auto"
|
||||
}
|
||||
if manager != "auto" && manager != "apt" && manager != "dnf" && manager != "yum" {
|
||||
return "", fmt.Errorf("unsupported --manager %q", opts.PackageManager)
|
||||
}
|
||||
ruleLine := ""
|
||||
if !opts.NoRule {
|
||||
addCmd := fmt.Sprintf("ip rule add to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Table)
|
||||
if opts.Priority > 0 {
|
||||
addCmd = fmt.Sprintf("ip rule add priority %d to \"$PXMON_GATEWAY_IP/32\" table %d", opts.Priority, opts.Table)
|
||||
}
|
||||
ruleLine = fmt.Sprintf(`
|
||||
if ! ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; then
|
||||
%s
|
||||
fi`, opts.Table, addCmd)
|
||||
}
|
||||
return fmt.Sprintf(`set -eu
|
||||
PXMON_GATEWAY_HOST=%s
|
||||
PXMON_GATEWAY_IP=%s
|
||||
PXMON_PROXY_URL=%s
|
||||
PXMON_MANAGER=%s
|
||||
if [ -z "$PXMON_GATEWAY_IP" ]; then
|
||||
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
|
||||
fi
|
||||
if [ -z "$PXMON_GATEWAY_IP" ]; then
|
||||
echo "cannot resolve repo gateway: $PXMON_GATEWAY_HOST" >&2
|
||||
exit 1
|
||||
fi
|
||||
%s
|
||||
pxmon_repo_manager="$PXMON_MANAGER"
|
||||
if [ "$pxmon_repo_manager" = "auto" ]; then
|
||||
if command -v apt-get >/dev/null 2>&1; then pxmon_repo_manager=apt
|
||||
elif command -v dnf >/dev/null 2>&1; then pxmon_repo_manager=dnf
|
||||
elif command -v yum >/dev/null 2>&1; then pxmon_repo_manager=yum
|
||||
else echo "no supported package manager found" >&2; exit 1
|
||||
fi
|
||||
fi
|
||||
case "$pxmon_repo_manager" in
|
||||
apt)
|
||||
mkdir -p /etc/apt/apt.conf.d
|
||||
cat > /etc/apt/apt.conf.d/99-pxmon-repo-tunnel <<EOF
|
||||
Acquire::http::Proxy "$PXMON_PROXY_URL";
|
||||
Acquire::https::Proxy "$PXMON_PROXY_URL";
|
||||
EOF
|
||||
;;
|
||||
dnf|yum)
|
||||
conf=/etc/dnf/dnf.conf
|
||||
[ "$pxmon_repo_manager" = "yum" ] && conf=/etc/yum.conf
|
||||
[ -f "$conf" ] || touch "$conf"
|
||||
awk '
|
||||
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
|
||||
/# END PXMON REPO TUNNEL/ {skip=0; next}
|
||||
!skip {print}
|
||||
' "$conf" > "$conf.tmp"
|
||||
mv "$conf.tmp" "$conf"
|
||||
{
|
||||
echo "# BEGIN PXMON REPO TUNNEL"
|
||||
echo "proxy=$PXMON_PROXY_URL"
|
||||
echo "# END PXMON REPO TUNNEL"
|
||||
} >> "$conf"
|
||||
;;
|
||||
esac
|
||||
echo "pxmon repo tunnel enabled: proxy=$PXMON_PROXY_URL gateway_ip=$PXMON_GATEWAY_IP manager=$pxmon_repo_manager"
|
||||
`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), shellQuote(gw.proxyURL), shellQuote(manager), ruleLine), nil
|
||||
}
|
||||
|
||||
func repoTunnelDisableScript(opts RepoTunnelOptions) (string, error) {
|
||||
gw, err := parseRepoTunnelGateway(opts.Gateway)
|
||||
if err != nil && !opts.NoRule {
|
||||
return "", err
|
||||
}
|
||||
ruleLine := ""
|
||||
if !opts.NoRule {
|
||||
if opts.Table <= 0 {
|
||||
return "", errors.New("--table is required unless --no-rule is used")
|
||||
}
|
||||
ruleLine = fmt.Sprintf(`
|
||||
PXMON_GATEWAY_HOST=%s
|
||||
PXMON_GATEWAY_IP=%s
|
||||
if [ -z "$PXMON_GATEWAY_IP" ]; then
|
||||
PXMON_GATEWAY_IP=$(getent ahostsv4 "$PXMON_GATEWAY_HOST" | awk '{print $1; exit}')
|
||||
fi
|
||||
if [ -n "$PXMON_GATEWAY_IP" ]; then
|
||||
while ip rule show | grep -Eq "to[[:space:]]+$PXMON_GATEWAY_IP(/32)?[[:space:]].*lookup[[:space:]]+%d\b"; do
|
||||
ip rule del to "$PXMON_GATEWAY_IP/32" table %d 2>/dev/null || break
|
||||
done
|
||||
fi`, shellQuote(gw.host), shellQuote(strings.TrimSpace(opts.GatewayIP)), opts.Table, opts.Table)
|
||||
}
|
||||
return fmt.Sprintf(`set -eu
|
||||
rm -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel
|
||||
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
|
||||
if [ -f "$conf" ]; then
|
||||
awk '
|
||||
/# BEGIN PXMON REPO TUNNEL/ {skip=1; next}
|
||||
/# END PXMON REPO TUNNEL/ {skip=0; next}
|
||||
!skip {print}
|
||||
' "$conf" > "$conf.tmp"
|
||||
mv "$conf.tmp" "$conf"
|
||||
fi
|
||||
done
|
||||
%s
|
||||
echo "pxmon repo tunnel disabled"
|
||||
`, ruleLine), nil
|
||||
}
|
||||
|
||||
func repoTunnelStatusScript() string {
|
||||
return `set -eu
|
||||
echo "== ip rules =="
|
||||
ip rule show | grep -E "lookup|table" || true
|
||||
echo
|
||||
echo "== apt proxy =="
|
||||
[ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ] && cat /etc/apt/apt.conf.d/99-pxmon-repo-tunnel || echo "(none)"
|
||||
echo
|
||||
echo "== dnf/yum proxy =="
|
||||
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
|
||||
[ -f "$conf" ] || continue
|
||||
echo "-- $conf"
|
||||
awk '/# BEGIN PXMON REPO TUNNEL/,/# END PXMON REPO TUNNEL/ {print}' "$conf"
|
||||
done`
|
||||
}
|
||||
|
||||
func repoTunnelDetectScript() string {
|
||||
return `set -eu
|
||||
if [ -f /etc/apt/apt.conf.d/99-pxmon-repo-tunnel ]; then
|
||||
proxy=$(sed -n 's/.*Proxy[[:space:]]*"\([^"]*\)".*/\1/p' /etc/apt/apt.conf.d/99-pxmon-repo-tunnel | head -1)
|
||||
[ -n "$proxy" ] && printf 'proxy=%s\nsource=apt\n' "$proxy" && exit 0
|
||||
fi
|
||||
for conf in /etc/dnf/dnf.conf /etc/yum.conf; do
|
||||
[ -f "$conf" ] || continue
|
||||
proxy=$(awk '
|
||||
/# BEGIN PXMON REPO TUNNEL/ {inside=1; next}
|
||||
/# END PXMON REPO TUNNEL/ {inside=0; next}
|
||||
inside && /^proxy[[:space:]]*=/ {
|
||||
sub(/^[^=]*=/, "")
|
||||
gsub(/^[[:space:]]+|[[:space:]]+$/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' "$conf")
|
||||
[ -n "$proxy" ] && printf 'proxy=%s\nsource=%s\n' "$proxy" "$conf" && exit 0
|
||||
done
|
||||
exit 0`
|
||||
}
|
||||
|
||||
type repoTunnelGateway struct {
|
||||
host string
|
||||
port int
|
||||
proxyURL string
|
||||
}
|
||||
|
||||
func parseRepoTunnelGateway(raw string) (repoTunnelGateway, error) {
|
||||
v := strings.TrimSpace(raw)
|
||||
if v == "" {
|
||||
return repoTunnelGateway{}, errors.New("--gateway is required")
|
||||
}
|
||||
if strings.HasPrefix(v, "http://") {
|
||||
v = strings.TrimPrefix(v, "http://")
|
||||
}
|
||||
if strings.HasPrefix(v, "https://") {
|
||||
return repoTunnelGateway{}, errors.New("--gateway must be an http proxy endpoint, not https")
|
||||
}
|
||||
host, portRaw, err := net.SplitHostPort(v)
|
||||
if err != nil {
|
||||
if strings.Count(v, ":") > 1 {
|
||||
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway %q; use host:port or [ipv6]:port", raw)
|
||||
}
|
||||
host = v
|
||||
portRaw = "3128"
|
||||
}
|
||||
host = strings.Trim(host, "[]")
|
||||
if strings.TrimSpace(host) == "" {
|
||||
return repoTunnelGateway{}, errors.New("--gateway host is empty")
|
||||
}
|
||||
port, err := strconv.Atoi(portRaw)
|
||||
if err != nil || port < 1 || port > 65535 {
|
||||
return repoTunnelGateway{}, fmt.Errorf("invalid --gateway port %q", portRaw)
|
||||
}
|
||||
return repoTunnelGateway{
|
||||
host: host,
|
||||
port: port,
|
||||
proxyURL: "http://" + net.JoinHostPort(host, strconv.Itoa(port)),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RunbookStep struct {
|
||||
Title string `json:"title"`
|
||||
Command string `json:"command,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
type Runbook struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Steps []RunbookStep `json:"steps"`
|
||||
BuiltIn bool `json:"built_in,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
func builtinRunbooks() []Runbook {
|
||||
return []Runbook{
|
||||
{
|
||||
ID: "vm-health-check",
|
||||
Name: "VM Health Check",
|
||||
Description: "Quick validation of agent, VM inventory and VM alert policy.",
|
||||
BuiltIn: true,
|
||||
Steps: []RunbookStep{
|
||||
{Title: "Check agent status", Command: "cluster agent status"},
|
||||
{Title: "Check cluster drift", Command: "cluster drift"},
|
||||
{Title: "Check VM states", Command: "cluster alert-vm check"},
|
||||
{Title: "Inspect VM allocations", Command: "kvm top"},
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "traffic-billing-audit",
|
||||
Name: "Traffic Billing Audit",
|
||||
Description: "Collect interface P95 and graph evidence for billing period.",
|
||||
BuiltIn: true,
|
||||
Steps: []RunbookStep{
|
||||
{Title: "List interfaces", Command: "cluster usage --range 1h"},
|
||||
{Title: "Compute P95 for target iface", Command: "cluster p95 --iface <iface> --range 30d --graph"},
|
||||
{Title: "Export report", Command: "cluster report export --format json --out ./report.json"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) runbookPath() string {
|
||||
return filepath.Join(s.DataDir(), "runbooks", "custom.json")
|
||||
}
|
||||
|
||||
func (s *Service) loadCustomRunbooks() ([]Runbook, error) {
|
||||
path := s.runbookPath()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []Runbook{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return []Runbook{}, nil
|
||||
}
|
||||
var items []Runbook
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Runbook, 0, len(items))
|
||||
for _, rb := range items {
|
||||
rb.ID = strings.TrimSpace(rb.ID)
|
||||
rb.Name = strings.TrimSpace(rb.Name)
|
||||
if rb.ID == "" || rb.Name == "" {
|
||||
continue
|
||||
}
|
||||
rb.BuiltIn = false
|
||||
out = append(out, rb)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Service) saveCustomRunbooks(items []Runbook) error {
|
||||
path := s.runbookPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func (s *Service) ListRunbooks() ([]Runbook, error) {
|
||||
built := builtinRunbooks()
|
||||
custom, err := s.loadCustomRunbooks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all := append(append([]Runbook{}, built...), custom...)
|
||||
sort.Slice(all, func(i, j int) bool { return strings.ToLower(all[i].ID) < strings.ToLower(all[j].ID) })
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func (s *Service) GetRunbook(selector string) (Runbook, bool) {
|
||||
selector = strings.TrimSpace(selector)
|
||||
items, err := s.ListRunbooks()
|
||||
if err != nil {
|
||||
return Runbook{}, false
|
||||
}
|
||||
for _, rb := range items {
|
||||
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
|
||||
return rb, true
|
||||
}
|
||||
}
|
||||
return Runbook{}, false
|
||||
}
|
||||
|
||||
func (s *Service) AddRunbook(rb Runbook) (Runbook, error) {
|
||||
rb.ID = strings.TrimSpace(rb.ID)
|
||||
rb.Name = strings.TrimSpace(rb.Name)
|
||||
if rb.ID == "" || rb.Name == "" {
|
||||
return Runbook{}, errors.New("runbook id and name are required")
|
||||
}
|
||||
if len(rb.Steps) == 0 {
|
||||
return Runbook{}, errors.New("runbook steps are required")
|
||||
}
|
||||
if b, ok := s.GetRunbook(rb.ID); ok && b.BuiltIn {
|
||||
return Runbook{}, errors.New("cannot overwrite built-in runbook")
|
||||
}
|
||||
custom, err := s.loadCustomRunbooks()
|
||||
if err != nil {
|
||||
return Runbook{}, err
|
||||
}
|
||||
for _, it := range custom {
|
||||
if strings.EqualFold(it.ID, rb.ID) {
|
||||
return Runbook{}, errors.New("runbook id already exists")
|
||||
}
|
||||
}
|
||||
now := s.now().UTC()
|
||||
rb.BuiltIn = false
|
||||
rb.CreatedAt = now
|
||||
rb.UpdatedAt = now
|
||||
custom = append(custom, rb)
|
||||
if err := s.saveCustomRunbooks(custom); err != nil {
|
||||
return Runbook{}, err
|
||||
}
|
||||
_ = s.AppendChange("runbook.add", rb.ID, rb.Name)
|
||||
return rb, nil
|
||||
}
|
||||
|
||||
func (s *Service) RemoveRunbook(selector string) (Runbook, error) {
|
||||
custom, err := s.loadCustomRunbooks()
|
||||
if err != nil {
|
||||
return Runbook{}, err
|
||||
}
|
||||
idx := -1
|
||||
for i, rb := range custom {
|
||||
if strings.EqualFold(rb.ID, selector) || strings.EqualFold(rb.Name, selector) {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return Runbook{}, errors.New("custom runbook not found")
|
||||
}
|
||||
removed := custom[idx]
|
||||
custom = append(custom[:idx], custom[idx+1:]...)
|
||||
if err := s.saveCustomRunbooks(custom); err != nil {
|
||||
return Runbook{}, err
|
||||
}
|
||||
_ = s.AppendChange("runbook.remove", removed.ID, removed.Name)
|
||||
return removed, nil
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ScheduledTask struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
Command string `json:"command"`
|
||||
Mode string `json:"mode,omitempty"` // shell|observer
|
||||
Every string `json:"every"`
|
||||
Backoff string `json:"backoff,omitempty"`
|
||||
JitterSec int `json:"jitter_sec,omitempty"`
|
||||
RetryMax int `json:"retry_max,omitempty"`
|
||||
RetryCur int `json:"retry_cur,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
LastRunAt time.Time `json:"last_run_at,omitempty"`
|
||||
NextRunAt time.Time `json:"next_run_at,omitempty"`
|
||||
}
|
||||
|
||||
type SchedulerRunResult struct {
|
||||
Task ScheduledTask `json:"task"`
|
||||
Ran bool `json:"ran"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Output string `json:"output,omitempty"`
|
||||
ExitCode int `json:"exit_code,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) schedulerPath() string {
|
||||
return filepath.Join(s.DataDir(), "scheduler", "tasks.json")
|
||||
}
|
||||
|
||||
func (s *Service) loadTasks() ([]ScheduledTask, error) {
|
||||
path := s.schedulerPath()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []ScheduledTask{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var items []ScheduledTask
|
||||
if len(strings.TrimSpace(string(raw))) == 0 {
|
||||
return []ScheduledTask{}, nil
|
||||
}
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range items {
|
||||
items[i] = normalizeScheduledTask(items[i])
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name) })
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Service) saveTasks(items []ScheduledTask) error {
|
||||
path := s.schedulerPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
b, err := json.MarshalIndent(items, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b = append(b, '\n')
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func normalizeScheduledTask(t ScheduledTask) ScheduledTask {
|
||||
t.ID = strings.TrimSpace(t.ID)
|
||||
t.Name = strings.TrimSpace(t.Name)
|
||||
t.Command = strings.TrimSpace(t.Command)
|
||||
t.Cluster = strings.TrimSpace(t.Cluster)
|
||||
t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
|
||||
t.Every = strings.TrimSpace(t.Every)
|
||||
if t.ID == "" {
|
||||
t.ID = newClusterID()
|
||||
}
|
||||
if t.Name == "" {
|
||||
t.Name = t.ID
|
||||
}
|
||||
if t.Every == "" {
|
||||
t.Every = "5m"
|
||||
}
|
||||
if strings.TrimSpace(t.Backoff) == "" {
|
||||
t.Backoff = "30s"
|
||||
}
|
||||
if t.JitterSec < 0 {
|
||||
t.JitterSec = 0
|
||||
}
|
||||
if t.RetryMax <= 0 {
|
||||
t.RetryMax = 3
|
||||
}
|
||||
if t.RetryCur < 0 {
|
||||
t.RetryCur = 0
|
||||
}
|
||||
if t.Mode == "" {
|
||||
t.Mode = "shell"
|
||||
}
|
||||
if t.Mode != "shell" && t.Mode != "observer" {
|
||||
t.Mode = "shell"
|
||||
}
|
||||
if t.CreatedAt.IsZero() {
|
||||
t.CreatedAt = time.Now().UTC()
|
||||
}
|
||||
if t.UpdatedAt.IsZero() {
|
||||
t.UpdatedAt = t.CreatedAt
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func parseTaskEvery(v string) (time.Duration, error) {
|
||||
d, err := time.ParseDuration(strings.TrimSpace(v))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid --every duration: %w", err)
|
||||
}
|
||||
if d < time.Minute {
|
||||
return 0, errors.New("--every must be >= 1m")
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerList() ([]ScheduledTask, error) {
|
||||
return s.loadTasks()
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerAdd(name, clusterSel, command, every, mode, backoff string, jitterSec, retryMax int, enabled bool) (ScheduledTask, error) {
|
||||
name = strings.TrimSpace(name)
|
||||
command = strings.TrimSpace(command)
|
||||
every = strings.TrimSpace(every)
|
||||
mode = strings.ToLower(strings.TrimSpace(mode))
|
||||
if name == "" {
|
||||
return ScheduledTask{}, errors.New("task name is required")
|
||||
}
|
||||
if command == "" {
|
||||
return ScheduledTask{}, errors.New("task command is required")
|
||||
}
|
||||
if mode == "" {
|
||||
mode = "shell"
|
||||
}
|
||||
if mode != "shell" && mode != "observer" {
|
||||
return ScheduledTask{}, errors.New("task mode must be shell|observer")
|
||||
}
|
||||
d, err := parseTaskEvery(every)
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
if strings.TrimSpace(backoff) == "" {
|
||||
backoff = "30s"
|
||||
}
|
||||
if _, err := time.ParseDuration(backoff); err != nil {
|
||||
return ScheduledTask{}, errors.New("invalid backoff duration")
|
||||
}
|
||||
if jitterSec < 0 {
|
||||
jitterSec = 0
|
||||
}
|
||||
if retryMax <= 0 {
|
||||
retryMax = 3
|
||||
}
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
for _, t := range items {
|
||||
if strings.EqualFold(t.Name, name) {
|
||||
return ScheduledTask{}, fmt.Errorf("task %q already exists", name)
|
||||
}
|
||||
}
|
||||
now := s.now().UTC()
|
||||
t := ScheduledTask{
|
||||
ID: newClusterID(),
|
||||
Name: name,
|
||||
Cluster: strings.TrimSpace(clusterSel),
|
||||
Command: command,
|
||||
Mode: mode,
|
||||
Every: every,
|
||||
Backoff: backoff,
|
||||
JitterSec: jitterSec,
|
||||
RetryMax: retryMax,
|
||||
Enabled: enabled,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
}
|
||||
if enabled {
|
||||
t.NextRunAt = now.Add(d)
|
||||
}
|
||||
items = append(items, t)
|
||||
if err := s.saveTasks(items); err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
_ = s.AppendChange("scheduler.add", name, fmt.Sprintf("%s mode=%s cluster=%s every=%s", command, mode, t.Cluster, every))
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerRemove(selector string) (ScheduledTask, error) {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
selector = strings.TrimSpace(selector)
|
||||
if selector == "" {
|
||||
return ScheduledTask{}, errors.New("task name or id is required")
|
||||
}
|
||||
idx := -1
|
||||
for i, t := range items {
|
||||
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx < 0 {
|
||||
return ScheduledTask{}, errors.New("task not found")
|
||||
}
|
||||
removed := items[idx]
|
||||
items = append(items[:idx], items[idx+1:]...)
|
||||
if err := s.saveTasks(items); err != nil {
|
||||
return ScheduledTask{}, err
|
||||
}
|
||||
_ = s.AppendChange("scheduler.remove", removed.Name, removed.Command)
|
||||
return removed, nil
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerMarkResult(taskID string, success bool, ranAt time.Time) error {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range items {
|
||||
if items[i].ID != taskID {
|
||||
continue
|
||||
}
|
||||
items[i].LastRunAt = ranAt.UTC()
|
||||
if success {
|
||||
d, err := parseTaskEvery(items[i].Every)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items[i].RetryCur = 0
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(d)
|
||||
} else {
|
||||
items[i].RetryCur++
|
||||
if items[i].RetryCur > items[i].RetryMax {
|
||||
// cap retries and move to the next normal run window
|
||||
items[i].RetryCur = 0
|
||||
d, err := parseTaskEvery(items[i].Every)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(d)
|
||||
} else {
|
||||
back, err := time.ParseDuration(strings.TrimSpace(items[i].Backoff))
|
||||
if err != nil || back <= 0 {
|
||||
back = 30 * time.Second
|
||||
}
|
||||
delay := back * time.Duration(1<<(items[i].RetryCur-1))
|
||||
if items[i].JitterSec > 0 {
|
||||
delay += time.Duration(rand.Intn(items[i].JitterSec+1)) * time.Second
|
||||
}
|
||||
items[i].NextRunAt = items[i].LastRunAt.Add(delay)
|
||||
}
|
||||
}
|
||||
items[i].UpdatedAt = ranAt.UTC()
|
||||
break
|
||||
}
|
||||
return s.saveTasks(items)
|
||||
}
|
||||
|
||||
func (s *Service) SchedulerDue(now time.Time) ([]ScheduledTask, error) {
|
||||
items, err := s.loadTasks()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n := now.UTC()
|
||||
due := make([]ScheduledTask, 0)
|
||||
for _, t := range items {
|
||||
if !t.Enabled {
|
||||
continue
|
||||
}
|
||||
if t.NextRunAt.IsZero() || !t.NextRunAt.After(n) {
|
||||
due = append(due, t)
|
||||
}
|
||||
}
|
||||
sort.Slice(due, func(i, j int) bool { return due[i].NextRunAt.Before(due[j].NextRunAt) })
|
||||
return due, nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,554 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newTestService(t *testing.T) *Service {
|
||||
t.Helper()
|
||||
|
||||
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore error: %v", err)
|
||||
}
|
||||
|
||||
return NewService(store)
|
||||
}
|
||||
|
||||
func TestConnectListUseDisconnect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
c1, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "eu-1",
|
||||
Host: "10.0.0.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass1",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("connect c1 error: %v", err)
|
||||
}
|
||||
|
||||
c2, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "us-1",
|
||||
Host: "10.0.0.11",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass2",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("connect c2 error: %v", err)
|
||||
}
|
||||
|
||||
clusters, activeID, err := svc.List()
|
||||
if err != nil {
|
||||
t.Fatalf("list error: %v", err)
|
||||
}
|
||||
if len(clusters) != 2 {
|
||||
t.Fatalf("expected 2 clusters, got %d", len(clusters))
|
||||
}
|
||||
if activeID != c2.ID {
|
||||
t.Fatalf("expected active %s, got %s", c2.ID, activeID)
|
||||
}
|
||||
|
||||
_, err = svc.Use(c1.Name)
|
||||
if err != nil {
|
||||
t.Fatalf("use error: %v", err)
|
||||
}
|
||||
|
||||
current, err := svc.Current()
|
||||
if err != nil {
|
||||
t.Fatalf("current error: %v", err)
|
||||
}
|
||||
if current.ID != c1.ID {
|
||||
t.Fatalf("expected current %s, got %s", c1.ID, current.ID)
|
||||
}
|
||||
|
||||
_, err = svc.Disconnect(c1.Name)
|
||||
if err != nil {
|
||||
t.Fatalf("disconnect error: %v", err)
|
||||
}
|
||||
|
||||
current, err = svc.Current()
|
||||
if err != nil {
|
||||
t.Fatalf("current after disconnect error: %v", err)
|
||||
}
|
||||
if current.ID != c2.ID {
|
||||
t.Fatalf("expected fallback current %s, got %s", c2.ID, current.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectDuplicateNameRequiresForce(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "prod",
|
||||
Host: "10.0.0.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass1",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("initial connect error: %v", err)
|
||||
}
|
||||
|
||||
_, _, err = svc.Connect(ctx, ConnectOptions{
|
||||
Name: "prod",
|
||||
Host: "10.0.0.20",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass2",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate name error")
|
||||
}
|
||||
|
||||
c, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "prod",
|
||||
Host: "10.0.0.20",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass2",
|
||||
SkipCheck: true,
|
||||
Force: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("force connect error: %v", err)
|
||||
}
|
||||
if c.Host != "10.0.0.20" {
|
||||
t.Fatalf("expected overwritten host, got %s", c.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportImportRestoresKeyFilesAndNewFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
srcDir := t.TempDir()
|
||||
keyPath := filepath.Join(srcDir, "id_ed25519")
|
||||
passPath := filepath.Join(srcDir, "pass.pxmonpassphrase")
|
||||
sftpKeyPath := filepath.Join(srcDir, "sftp_key")
|
||||
if err := os.WriteFile(keyPath, []byte("PRIVATE KEY\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(passPath, []byte("secret-pass\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(sftpKeyPath, []byte("SFTP PRIVATE KEY\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
src := newTestService(t)
|
||||
reg := newRegistry()
|
||||
reg.ActiveClusterID = "clu_1"
|
||||
reg.Backups.Targets = []BackupTarget{{
|
||||
ID: "bt_1",
|
||||
Name: "sftp",
|
||||
Type: "sftp",
|
||||
Enabled: true,
|
||||
SFTPKeyPath: sftpKeyPath,
|
||||
}}
|
||||
reg.Clusters = []Cluster{{
|
||||
ID: "clu_1",
|
||||
Name: "node",
|
||||
Host: "192.0.2.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
Transport: TransportIPFabric,
|
||||
AuthMethod: AuthMethodKey,
|
||||
KeyPath: keyPath,
|
||||
KeyPassphraseFile: passPath,
|
||||
RepoTunnel: RepoTunnelState{
|
||||
Enabled: true,
|
||||
Proxy: "http://203.0.113.10:3128",
|
||||
Source: "dnf",
|
||||
},
|
||||
Alerts: defaultAlertPolicy(),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}}
|
||||
if err := src.store.Save(reg); err != nil {
|
||||
t.Fatalf("save source registry: %v", err)
|
||||
}
|
||||
|
||||
bundlePath := filepath.Join(t.TempDir(), "pxmon-export.enc")
|
||||
if err := src.Export(bundlePath, "strong-test-passphrase"); err != nil {
|
||||
t.Fatalf("export error: %v", err)
|
||||
}
|
||||
raw, err := os.ReadFile(bundlePath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(string(raw), "PRIVATE KEY") || strings.Contains(string(raw), "secret-pass") {
|
||||
t.Fatal("export bundle leaked key material in plaintext")
|
||||
}
|
||||
|
||||
dst := newTestService(t)
|
||||
if _, err := dst.Import(bundlePath, "strong-test-passphrase", ImportModeReplace); err != nil {
|
||||
t.Fatalf("import error: %v", err)
|
||||
}
|
||||
gotReg, err := dst.store.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load imported registry: %v", err)
|
||||
}
|
||||
if len(gotReg.Clusters) != 1 {
|
||||
t.Fatalf("expected 1 cluster, got %d", len(gotReg.Clusters))
|
||||
}
|
||||
got := gotReg.Clusters[0]
|
||||
if !got.RepoTunnel.Enabled || got.RepoTunnel.Proxy != "http://203.0.113.10:3128" {
|
||||
t.Fatalf("repo tunnel was not preserved: %+v", got.RepoTunnel)
|
||||
}
|
||||
if got.KeyPath == keyPath || got.KeyPassphraseFile == passPath {
|
||||
t.Fatalf("expected key paths to be restored under destination config dir, got key=%q pass=%q", got.KeyPath, got.KeyPassphraseFile)
|
||||
}
|
||||
keyData, err := os.ReadFile(got.KeyPath)
|
||||
if err != nil {
|
||||
t.Fatalf("read restored key: %v", err)
|
||||
}
|
||||
if string(keyData) != "PRIVATE KEY\n" {
|
||||
t.Fatalf("unexpected restored key content: %q", string(keyData))
|
||||
}
|
||||
passData, err := os.ReadFile(got.KeyPassphraseFile)
|
||||
if err != nil {
|
||||
t.Fatalf("read restored passphrase file: %v", err)
|
||||
}
|
||||
if string(passData) != "secret-pass\n" {
|
||||
t.Fatalf("unexpected restored passphrase content: %q", string(passData))
|
||||
}
|
||||
if len(gotReg.Backups.Targets) != 1 || gotReg.Backups.Targets[0].SFTPKeyPath == sftpKeyPath {
|
||||
t.Fatalf("expected restored SFTP key path, got %+v", gotReg.Backups.Targets)
|
||||
}
|
||||
|
||||
mergeDst := newTestService(t)
|
||||
if _, err := mergeDst.Import(bundlePath, "strong-test-passphrase", ImportModeMerge); err != nil {
|
||||
t.Fatalf("merge import error: %v", err)
|
||||
}
|
||||
mergeReg, err := mergeDst.store.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load merge registry: %v", err)
|
||||
}
|
||||
if len(mergeReg.Backups.Targets) != 1 || strings.TrimSpace(mergeReg.Backups.Targets[0].SFTPKeyPath) == "" {
|
||||
t.Fatalf("expected backup target to be merged, got %+v", mergeReg.Backups)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectRequiresAuthData(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "bad",
|
||||
Host: "10.0.0.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty password auth")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlertPolicySetAndGet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
_, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "node-1",
|
||||
Host: "10.0.0.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("connect error: %v", err)
|
||||
}
|
||||
|
||||
updated, err := svc.SetAlertPolicy("node-1", AlertPolicy{
|
||||
CPUWarnPercent: 70,
|
||||
RAMWarnPercent: 75,
|
||||
SwapWarnPercent: 60,
|
||||
DiskWarnPercent: 80,
|
||||
NetWarnMbps: 120,
|
||||
NetSustainEnabled: true,
|
||||
NetSustainIface: "eth0",
|
||||
NetSustainInclude: []string{"net0"},
|
||||
NetSustainExclude: []string{"backup"},
|
||||
NetSustainMbps: 500,
|
||||
NetSustainMinutes: 60,
|
||||
NetSustainCooldownMins: 15,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("set alert policy error: %v", err)
|
||||
}
|
||||
|
||||
if updated.Alerts.NetWarnMbps != 120 {
|
||||
t.Fatalf("unexpected net threshold: %.2f", updated.Alerts.NetWarnMbps)
|
||||
}
|
||||
|
||||
got, err := svc.GetAlertPolicy("node-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get alert policy error: %v", err)
|
||||
}
|
||||
if got.RAMWarnPercent != 75 || got.DiskWarnPercent != 80 {
|
||||
t.Fatalf("unexpected thresholds: %+v", got)
|
||||
}
|
||||
if !got.NetSustainEnabled || got.NetSustainIface != "eth0" || got.NetSustainMbps != 500 {
|
||||
t.Fatalf("unexpected sustained net policy: %+v", got)
|
||||
}
|
||||
if len(got.NetSustainInclude) != 1 || got.NetSustainInclude[0] != "net0" {
|
||||
t.Fatalf("unexpected sustained net include filter: %+v", got.NetSustainInclude)
|
||||
}
|
||||
if len(got.NetSustainExclude) != 1 || got.NetSustainExclude[0] != "backup" {
|
||||
t.Fatalf("unexpected sustained net exclude filter: %+v", got.NetSustainExclude)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPluginToolSupported(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
info := SoftwareInfo{
|
||||
Bird: true,
|
||||
FRR: true,
|
||||
KVM: true,
|
||||
LXC: true,
|
||||
LXD: false,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
tool string
|
||||
ok bool
|
||||
}{
|
||||
{tool: "bird", ok: true},
|
||||
{tool: "frr", ok: true},
|
||||
{tool: "kvm", ok: true},
|
||||
{tool: "lxc", ok: true},
|
||||
{tool: "lxd", ok: true}, // lxd aliases to lxc command templates
|
||||
{tool: "unknown", ok: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.tool, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
if got := isPluginToolSupported(info, tt.tool); got != tt.ok {
|
||||
t.Fatalf("tool=%s expected %v got %v", tt.tool, tt.ok, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLXDTopUsesTemplateNotRawLxcTop(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script, err := pluginScript("lxd", "top", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("pluginScript error: %v", err)
|
||||
}
|
||||
if strings.Contains(script, "lxc top") {
|
||||
t.Fatalf("expected custom template script, got raw lxc top: %q", script)
|
||||
}
|
||||
if !strings.Contains(script, "lxc info") {
|
||||
t.Fatalf("expected lxc info usage in top template")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPluginActionReportsMissingSupport(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
ctx := context.Background()
|
||||
|
||||
cluster, _, err := svc.Connect(ctx, ConnectOptions{
|
||||
Name: "eu-1",
|
||||
Host: "127.0.0.1",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "pass",
|
||||
SkipCheck: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("connect error: %v", err)
|
||||
}
|
||||
|
||||
reg, err := svc.store.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("load registry error: %v", err)
|
||||
}
|
||||
for i := range reg.Clusters {
|
||||
if reg.Clusters[i].ID == cluster.ID {
|
||||
reg.Clusters[i].Software = SoftwareInfo{
|
||||
DetectedAt: time.Now().UTC(),
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if err := svc.store.Save(reg); err != nil {
|
||||
t.Fatalf("save registry error: %v", err)
|
||||
}
|
||||
|
||||
_, err = svc.RunPluginAction(ctx, cluster.Name, "lxd", "top", nil)
|
||||
if err == nil {
|
||||
t.Fatalf("expected unsupported software error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "support for lxd was not detected") {
|
||||
t.Fatalf("unexpected error message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVMTopScriptIncludesReadableMetrics(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script, err := pluginScript("kvm", "top", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("pluginScript error: %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"VCPU",
|
||||
"RAM_MAX",
|
||||
"DISK_CAP",
|
||||
"DISK_ALLOC",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("expected %q in kvm top script", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVMNetTopScriptIncludesRateAndP95(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script, err := pluginScript("kvm", "net-top", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("pluginScript error: %v", err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"NET_Mbps",
|
||||
"RX_Mbps",
|
||||
"TX_Mbps",
|
||||
"P95_Mbps",
|
||||
"RX_TOTAL",
|
||||
"TX_TOTAL",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Fatalf("expected %q in kvm net-top script", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVMScriptsAreShellParseable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
tool string
|
||||
action string
|
||||
}{
|
||||
{tool: "kvm", action: "top"},
|
||||
{tool: "kvm", action: "net-top"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.tool+"-"+tc.action, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script, err := pluginScript(tc.tool, tc.action, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("pluginScript error: %v", err)
|
||||
}
|
||||
|
||||
cmd := exec.Command("sh", "-n", "-c", script)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("shell parse failed: %v\n%s\nSCRIPT:\n%s", err, string(out), script)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramConfigSetGetDisable(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
cfg, err := svc.SetTelegram(Telegram{
|
||||
Enabled: true,
|
||||
Token: "123:ABC",
|
||||
AllowedUserIDs: []int64{2002, 1001, 2002},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("set telegram config: %v", err)
|
||||
}
|
||||
if !cfg.Enabled {
|
||||
t.Fatal("expected enabled telegram config")
|
||||
}
|
||||
if len(cfg.AllowedUserIDs) != 2 {
|
||||
t.Fatalf("expected deduped ids, got %+v", cfg.AllowedUserIDs)
|
||||
}
|
||||
|
||||
got, err := svc.GetTelegram()
|
||||
if err != nil {
|
||||
t.Fatalf("get telegram config: %v", err)
|
||||
}
|
||||
if got.Token != "123:ABC" {
|
||||
t.Fatalf("unexpected token: %q", got.Token)
|
||||
}
|
||||
if len(got.AllowedUserIDs) != 2 || got.AllowedUserIDs[0] != 1001 || got.AllowedUserIDs[1] != 2002 {
|
||||
t.Fatalf("unexpected allowed ids: %+v", got.AllowedUserIDs)
|
||||
}
|
||||
|
||||
disabled, err := svc.DisableTelegram()
|
||||
if err != nil {
|
||||
t.Fatalf("disable telegram config: %v", err)
|
||||
}
|
||||
if disabled.Enabled {
|
||||
t.Fatal("expected disabled telegram config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelegramConfigValidation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
svc := newTestService(t)
|
||||
_, err := svc.SetTelegram(Telegram{
|
||||
Enabled: true,
|
||||
AllowedUserIDs: []int64{123},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected token validation error")
|
||||
}
|
||||
|
||||
_, err = svc.SetTelegram(Telegram{
|
||||
Enabled: true,
|
||||
Token: "123:ABC",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected allowed ids validation error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pxmon/internal/history"
|
||||
)
|
||||
|
||||
type AvailabilityVM struct {
|
||||
Name string `json:"name"`
|
||||
Availability float64 `json:"availability_pct"`
|
||||
Samples int `json:"samples"`
|
||||
Running int `json:"running_samples"`
|
||||
}
|
||||
|
||||
type AvailabilityReport struct {
|
||||
Cluster string `json:"cluster"`
|
||||
Range string `json:"range"`
|
||||
Samples int `json:"samples"`
|
||||
UpSamples int `json:"up_samples"`
|
||||
Availability float64 `json:"availability_pct"`
|
||||
VMs []AvailabilityVM `json:"vms,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) AvailabilityReport(selector string, since time.Time, vmFilter string) (AvailabilityReport, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return AvailabilityReport{}, err
|
||||
}
|
||||
store := history.NewAvailabilityStore(s.DataDir())
|
||||
snaps, err := store.Load(c.ID, since)
|
||||
if err != nil {
|
||||
return AvailabilityReport{}, err
|
||||
}
|
||||
rep := AvailabilityReport{Cluster: c.Name, Samples: len(snaps)}
|
||||
if len(snaps) == 0 {
|
||||
return rep, nil
|
||||
}
|
||||
vmFilter = strings.TrimSpace(vmFilter)
|
||||
totalUp := 0
|
||||
type acc struct{ samples, running int }
|
||||
vmap := map[string]*acc{}
|
||||
for _, snap := range snaps {
|
||||
if snap.ClusterUp {
|
||||
totalUp++
|
||||
}
|
||||
for vm, st := range snap.VMStates {
|
||||
if vmFilter != "" && !strings.EqualFold(vmFilter, vm) {
|
||||
continue
|
||||
}
|
||||
a := vmap[vm]
|
||||
if a == nil {
|
||||
a = &acc{}
|
||||
vmap[vm] = a
|
||||
}
|
||||
a.samples++
|
||||
if strings.EqualFold(strings.TrimSpace(st), "running") {
|
||||
a.running++
|
||||
}
|
||||
}
|
||||
}
|
||||
rep.UpSamples = totalUp
|
||||
rep.Availability = 100 * float64(totalUp) / float64(len(snaps))
|
||||
for vm, a := range vmap {
|
||||
if a.samples == 0 {
|
||||
continue
|
||||
}
|
||||
rep.VMs = append(rep.VMs, AvailabilityVM{
|
||||
Name: vm,
|
||||
Samples: a.samples,
|
||||
Running: a.running,
|
||||
Availability: 100 * float64(a.running) / float64(a.samples),
|
||||
})
|
||||
}
|
||||
sort.Slice(rep.VMs, func(i, j int) bool { return rep.VMs[i].Name < rep.VMs[j].Name })
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
type CapacityForecastItem struct {
|
||||
Mount string `json:"mount"`
|
||||
UsedPct float64 `json:"used_pct"`
|
||||
SlopeBytesSec float64 `json:"slope_bytes_per_sec"`
|
||||
DaysTo90 float64 `json:"days_to_90_pct"`
|
||||
DaysTo95 float64 `json:"days_to_95_pct"`
|
||||
}
|
||||
|
||||
type CapacityForecastReport struct {
|
||||
Cluster string `json:"cluster"`
|
||||
Samples int `json:"samples"`
|
||||
Items []CapacityForecastItem `json:"items,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Service) CapacityForecast(selector string, since time.Time) (CapacityForecastReport, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return CapacityForecastReport{}, err
|
||||
}
|
||||
store := history.NewCapacityStore(s.DataDir())
|
||||
snaps, err := store.Load(c.ID, since)
|
||||
if err != nil {
|
||||
return CapacityForecastReport{}, err
|
||||
}
|
||||
rep := CapacityForecastReport{Cluster: c.Name, Samples: len(snaps)}
|
||||
if len(snaps) < 2 {
|
||||
return rep, nil
|
||||
}
|
||||
type point struct {
|
||||
ts time.Time
|
||||
used float64
|
||||
tot float64
|
||||
}
|
||||
byMount := map[string][]point{}
|
||||
for _, snap := range snaps {
|
||||
for _, d := range snap.Disks {
|
||||
if d.TotalBytes == 0 {
|
||||
continue
|
||||
}
|
||||
byMount[d.Mount] = append(byMount[d.Mount], point{ts: snap.Timestamp, used: float64(d.UsedBytes), tot: float64(d.TotalBytes)})
|
||||
}
|
||||
}
|
||||
for mnt, pts := range byMount {
|
||||
if len(pts) < 2 {
|
||||
continue
|
||||
}
|
||||
sort.Slice(pts, func(i, j int) bool { return pts[i].ts.Before(pts[j].ts) })
|
||||
first := pts[0]
|
||||
last := pts[len(pts)-1]
|
||||
dt := last.ts.Sub(first.ts).Seconds()
|
||||
if dt <= 0 {
|
||||
continue
|
||||
}
|
||||
slope := (last.used - first.used) / dt
|
||||
usedPct := 100 * last.used / last.tot
|
||||
d90 := daysToTarget(last.used, last.tot*0.90, slope)
|
||||
d95 := daysToTarget(last.used, last.tot*0.95, slope)
|
||||
rep.Items = append(rep.Items, CapacityForecastItem{
|
||||
Mount: mnt,
|
||||
UsedPct: usedPct,
|
||||
SlopeBytesSec: slope,
|
||||
DaysTo90: d90,
|
||||
DaysTo95: d95,
|
||||
})
|
||||
}
|
||||
sort.Slice(rep.Items, func(i, j int) bool { return rep.Items[i].UsedPct > rep.Items[j].UsedPct })
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func daysToTarget(current, target, slope float64) float64 {
|
||||
if target <= current {
|
||||
return 0
|
||||
}
|
||||
if slope <= 0 {
|
||||
return math.Inf(1)
|
||||
}
|
||||
return (target - current) / slope / 86400
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
envConfigPath = "PXMON_CONFIG"
|
||||
envMasterKeyPath = "PXMON_MASTER_KEY"
|
||||
filePrefix = "OBSCTL1:"
|
||||
masterKeyBytes = 32
|
||||
)
|
||||
|
||||
// Store persists encrypted cluster registry on disk.
|
||||
type Store struct {
|
||||
path string
|
||||
keyPath string
|
||||
}
|
||||
|
||||
func NewStore(path string) (*Store, error) {
|
||||
if path == "" {
|
||||
var err error
|
||||
path, err = DefaultConfigPath()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
keyPath, err := defaultMasterKeyPath(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Store{path: path, keyPath: keyPath}, nil
|
||||
}
|
||||
|
||||
func DefaultConfigPath() (string, error) {
|
||||
if p := os.Getenv(envConfigPath); p != "" {
|
||||
return p, nil
|
||||
}
|
||||
|
||||
dir, err := os.UserConfigDir()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("resolve user config dir: %w", err)
|
||||
}
|
||||
|
||||
return filepath.Join(dir, "pxmon", "clusters.enc"), nil
|
||||
}
|
||||
|
||||
func defaultMasterKeyPath(configPath string) (string, error) {
|
||||
if p := os.Getenv(envMasterKeyPath); p != "" {
|
||||
return p, nil
|
||||
}
|
||||
if configPath == "" {
|
||||
return "", errors.New("empty config path")
|
||||
}
|
||||
return filepath.Join(filepath.Dir(configPath), "master.key"), nil
|
||||
}
|
||||
|
||||
func (s *Store) Path() string {
|
||||
return s.path
|
||||
}
|
||||
|
||||
func (s *Store) KeyPath() string {
|
||||
return s.keyPath
|
||||
}
|
||||
|
||||
func (s *Store) LockerSessionPath() string {
|
||||
return filepath.Join(filepath.Dir(s.path), "locker.session")
|
||||
}
|
||||
|
||||
func (s *Store) LockerAuditPath() string {
|
||||
return filepath.Join(filepath.Dir(s.path), "locker.audit.log")
|
||||
}
|
||||
|
||||
func (s *Store) Load() (Registry, error) {
|
||||
f, err := os.Open(s.path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return newRegistry(), nil
|
||||
}
|
||||
return Registry{}, fmt.Errorf("open registry file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
data, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return Registry{}, fmt.Errorf("read registry file: %w", err)
|
||||
}
|
||||
if len(strings.TrimSpace(string(data))) == 0 {
|
||||
return newRegistry(), nil
|
||||
}
|
||||
|
||||
payload, err := s.decodePayload(data)
|
||||
if err != nil {
|
||||
return Registry{}, err
|
||||
}
|
||||
|
||||
var reg Registry
|
||||
if err := json.Unmarshal(payload, ®); err != nil {
|
||||
return Registry{}, fmt.Errorf("decode registry JSON: %w", err)
|
||||
}
|
||||
|
||||
if reg.Version == 0 {
|
||||
reg.Version = currentVersion
|
||||
}
|
||||
if reg.Clusters == nil {
|
||||
reg.Clusters = []Cluster{}
|
||||
}
|
||||
for i := range reg.Clusters {
|
||||
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
|
||||
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
|
||||
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
|
||||
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
|
||||
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
|
||||
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
|
||||
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
|
||||
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
|
||||
}
|
||||
reg.Telegram = normalizeTelegram(reg.Telegram)
|
||||
reg.Locker = normalizeLocker(reg.Locker)
|
||||
reg.Backups = normalizeBackupConfig(reg.Backups)
|
||||
|
||||
return reg, nil
|
||||
}
|
||||
|
||||
func (s *Store) decodePayload(data []byte) ([]byte, error) {
|
||||
text := strings.TrimSpace(string(data))
|
||||
|
||||
if strings.HasPrefix(text, "{") {
|
||||
// Backward compatibility with legacy unencrypted format.
|
||||
return []byte(text), nil
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(text, filePrefix) {
|
||||
return nil, errors.New("unsupported registry format")
|
||||
}
|
||||
|
||||
blob := strings.TrimPrefix(text, filePrefix)
|
||||
raw, err := base64.StdEncoding.DecodeString(blob)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode encrypted payload: %w", err)
|
||||
}
|
||||
|
||||
key, err := s.loadOrCreateMasterKey()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
payload, err := decrypt(raw, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt registry: %w", err)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (s *Store) Save(reg Registry) error {
|
||||
reg.Version = currentVersion
|
||||
if reg.Clusters == nil {
|
||||
reg.Clusters = []Cluster{}
|
||||
}
|
||||
for i := range reg.Clusters {
|
||||
reg.Clusters[i].Alerts = ensureAlertPolicy(reg.Clusters[i].Alerts)
|
||||
reg.Clusters[i].VMAlerts = ensureVMAlertPolicy(reg.Clusters[i].VMAlerts)
|
||||
reg.Clusters[i].AlertRouting = ensureAlertRoutingPolicy(reg.Clusters[i].AlertRouting)
|
||||
reg.Clusters[i].RunbookTrigger = ensureRunbookTrigger(reg.Clusters[i].RunbookTrigger)
|
||||
reg.Clusters[i].Drift = normalizeDriftControl(reg.Clusters[i].Drift)
|
||||
reg.Clusters[i].Tags = normalizeTagList(reg.Clusters[i].Tags)
|
||||
reg.Clusters[i].KVMTags = normalizeVMTagMap(reg.Clusters[i].KVMTags)
|
||||
reg.Clusters[i].Transport = normalizeTransport(reg.Clusters[i].Transport)
|
||||
}
|
||||
reg.Telegram = normalizeTelegram(reg.Telegram)
|
||||
reg.Locker = normalizeLocker(reg.Locker)
|
||||
reg.Backups = normalizeBackupConfig(reg.Backups)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
|
||||
return fmt.Errorf("create config dir: %w", err)
|
||||
}
|
||||
|
||||
payload, err := json.MarshalIndent(reg, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode registry JSON: %w", err)
|
||||
}
|
||||
|
||||
key, err := s.loadOrCreateMasterKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
encrypted, err := encrypt(payload, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encrypt registry: %w", err)
|
||||
}
|
||||
|
||||
content := filePrefix + base64.StdEncoding.EncodeToString(encrypted) + "\n"
|
||||
|
||||
tmp := s.path + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(content), 0o600); err != nil {
|
||||
return fmt.Errorf("write temp registry file: %w", err)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmp, s.path); err != nil {
|
||||
return fmt.Errorf("replace registry file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) loadOrCreateMasterKey() ([]byte, error) {
|
||||
if err := os.MkdirAll(filepath.Dir(s.keyPath), 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create key dir: %w", err)
|
||||
}
|
||||
|
||||
key, err := os.ReadFile(s.keyPath)
|
||||
if err == nil {
|
||||
if len(key) != masterKeyBytes {
|
||||
return nil, fmt.Errorf("invalid master key length: got %d", len(key))
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
return nil, fmt.Errorf("read master key: %w", err)
|
||||
}
|
||||
|
||||
key = make([]byte, masterKeyBytes)
|
||||
if _, err := rand.Read(key); err != nil {
|
||||
return nil, fmt.Errorf("generate master key: %w", err)
|
||||
}
|
||||
|
||||
tmp := s.keyPath + ".tmp"
|
||||
if err := os.WriteFile(tmp, key, 0o600); err != nil {
|
||||
return nil, fmt.Errorf("write temp master key: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, s.keyPath); err != nil {
|
||||
return nil, fmt.Errorf("replace master key: %w", err)
|
||||
}
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func encrypt(payload, key []byte) ([]byte, error) {
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nonce := make([]byte, gcm.NonceSize())
|
||||
if _, err := rand.Read(nonce); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sealed := gcm.Seal(nil, nonce, payload, nil)
|
||||
out := make([]byte, 0, len(nonce)+len(sealed))
|
||||
out = append(out, nonce...)
|
||||
out = append(out, sealed...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func decrypt(raw, key []byte) ([]byte, error) {
|
||||
block, err := aes.NewCipher(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gcm, err := cipher.NewGCM(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nonceSize := gcm.NonceSize()
|
||||
if len(raw) <= nonceSize {
|
||||
return nil, errors.New("ciphertext too short")
|
||||
}
|
||||
|
||||
nonce := raw[:nonceSize]
|
||||
ciphertext := raw[nonceSize:]
|
||||
|
||||
payload, err := gcm.Open(nil, nonce, ciphertext, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
type lockerSessionState struct {
|
||||
ExpiresAtUnix int64 `json:"expires_at_unix"`
|
||||
SigHex string `json:"sig_hex"`
|
||||
}
|
||||
|
||||
func (s *Store) SaveLockerSession(passwordHash string, ttl time.Duration) error {
|
||||
passwordHash = strings.TrimSpace(passwordHash)
|
||||
if passwordHash == "" {
|
||||
return errors.New("empty locker password hash")
|
||||
}
|
||||
if ttl <= 0 {
|
||||
ttl = 6 * time.Hour
|
||||
}
|
||||
|
||||
key, err := s.loadOrCreateMasterKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
expires := time.Now().UTC().Add(ttl).Unix()
|
||||
payload := strconv.FormatInt(expires, 10) + "|" + passwordHash
|
||||
mac := hmac.New(sha256.New, key)
|
||||
_, _ = mac.Write([]byte(payload))
|
||||
sig := hex.EncodeToString(mac.Sum(nil))
|
||||
|
||||
state := lockerSessionState{
|
||||
ExpiresAtUnix: expires,
|
||||
SigHex: sig,
|
||||
}
|
||||
data, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
|
||||
path := s.LockerSessionPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
func (s *Store) ValidateLockerSession(passwordHash string) (bool, time.Time, error) {
|
||||
passwordHash = strings.TrimSpace(passwordHash)
|
||||
if passwordHash == "" {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
|
||||
path := s.LockerSessionPath()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
|
||||
var state lockerSessionState
|
||||
if err := json.Unmarshal(raw, &state); err != nil {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
if state.ExpiresAtUnix <= 0 || strings.TrimSpace(state.SigHex) == "" {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
expiresAt := time.Unix(state.ExpiresAtUnix, 0).UTC()
|
||||
if time.Now().UTC().After(expiresAt) {
|
||||
return false, expiresAt, nil
|
||||
}
|
||||
|
||||
key, err := s.loadOrCreateMasterKey()
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
|
||||
payload := strconv.FormatInt(state.ExpiresAtUnix, 10) + "|" + passwordHash
|
||||
mac := hmac.New(sha256.New, key)
|
||||
_, _ = mac.Write([]byte(payload))
|
||||
expected := mac.Sum(nil)
|
||||
got, err := hex.DecodeString(strings.TrimSpace(state.SigHex))
|
||||
if err != nil {
|
||||
return false, expiresAt, nil
|
||||
}
|
||||
if !hmac.Equal(expected, got) {
|
||||
return false, expiresAt, nil
|
||||
}
|
||||
return true, expiresAt, nil
|
||||
}
|
||||
|
||||
func (s *Store) ClearLockerSession() error {
|
||||
path := s.LockerSessionPath()
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) AppendLockerAudit(event, detail string) error {
|
||||
path := s.LockerAuditPath()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
ts := time.Now().UTC().Format(time.RFC3339)
|
||||
event = strings.TrimSpace(event)
|
||||
detail = strings.TrimSpace(detail)
|
||||
if event == "" {
|
||||
event = "event"
|
||||
}
|
||||
if detail == "" {
|
||||
detail = "-"
|
||||
}
|
||||
_, err = fmt.Fprintf(f, "%s event=%s detail=%s\n", ts, event, strconv.Quote(detail))
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStoreLoadMissing(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
store, err := NewStore(filepath.Join(t.TempDir(), "clusters.enc"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore error: %v", err)
|
||||
}
|
||||
|
||||
reg, err := store.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load error: %v", err)
|
||||
}
|
||||
|
||||
if reg.Version != currentVersion {
|
||||
t.Fatalf("expected version %d, got %d", currentVersion, reg.Version)
|
||||
}
|
||||
if len(reg.Clusters) != 0 {
|
||||
t.Fatalf("expected empty clusters, got %d", len(reg.Clusters))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreSaveLoadRoundtrip(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
path := filepath.Join(t.TempDir(), "clusters.enc")
|
||||
store, err := NewStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore error: %v", err)
|
||||
}
|
||||
|
||||
seed := newRegistry()
|
||||
seed.ActiveClusterID = "clu_1"
|
||||
seed.Clusters = []Cluster{{
|
||||
ID: "clu_1",
|
||||
Name: "prod",
|
||||
Host: "10.0.0.10",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "secret123",
|
||||
}}
|
||||
|
||||
if err := store.Save(seed); err != nil {
|
||||
t.Fatalf("Save error: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load error: %v", err)
|
||||
}
|
||||
|
||||
if got.ActiveClusterID != "clu_1" {
|
||||
t.Fatalf("active cluster mismatch: %s", got.ActiveClusterID)
|
||||
}
|
||||
if len(got.Clusters) != 1 || got.Clusters[0].Name != "prod" {
|
||||
t.Fatalf("unexpected clusters: %+v", got.Clusters)
|
||||
}
|
||||
if got.Clusters[0].Password != "secret123" {
|
||||
t.Fatalf("password mismatch after decrypt: %s", got.Clusters[0].Password)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorePersistsEncryptedPayload(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "clusters.enc")
|
||||
store, err := NewStore(path)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStore error: %v", err)
|
||||
}
|
||||
|
||||
reg := newRegistry()
|
||||
reg.Clusters = []Cluster{{
|
||||
ID: "clu_1",
|
||||
Name: "sensitive-prod",
|
||||
Host: "192.168.1.1",
|
||||
Port: 22,
|
||||
User: "root",
|
||||
AuthMethod: AuthMethodPassword,
|
||||
Password: "very-secret",
|
||||
}}
|
||||
if err := store.Save(reg); err != nil {
|
||||
t.Fatalf("Save error: %v", err)
|
||||
}
|
||||
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile error: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(string(raw), filePrefix) {
|
||||
t.Fatalf("expected encrypted file prefix %q", filePrefix)
|
||||
}
|
||||
if strings.Contains(string(raw), "sensitive-prod") || strings.Contains(string(raw), "very-secret") {
|
||||
t.Fatal("plaintext secrets leaked into encrypted file")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(store.KeyPath()); err != nil {
|
||||
t.Fatalf("master key not created: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (s *Service) AddClusterTags(selector string, tags []string) (Cluster, error) {
|
||||
return s.updateClusterTags(selector, tags, true)
|
||||
}
|
||||
|
||||
func (s *Service) RemoveClusterTags(selector string, tags []string) (Cluster, error) {
|
||||
return s.updateClusterTags(selector, tags, false)
|
||||
}
|
||||
|
||||
func (s *Service) updateClusterTags(selector string, tags []string, add bool) (Cluster, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
changes := make(map[string]struct{}, len(tags))
|
||||
for _, t := range normalizeTagList(tags) {
|
||||
changes[t] = struct{}{}
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
return c, errors.New("at least one non-empty tag is required")
|
||||
}
|
||||
|
||||
current := make(map[string]struct{}, len(c.Tags))
|
||||
for _, t := range normalizeTagList(c.Tags) {
|
||||
current[t] = struct{}{}
|
||||
}
|
||||
if add {
|
||||
for t := range changes {
|
||||
current[t] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
for t := range changes {
|
||||
delete(current, t)
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(current))
|
||||
for t := range current {
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Strings(out)
|
||||
c.Tags = out
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("cluster.tags", c.Name, strings.Join(out, ","))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListClusterTags(selector string) ([]string, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return normalizeTagList(c.Tags), nil
|
||||
}
|
||||
|
||||
func (s *Service) AddKVMTag(selector, vm string, tags []string) (Cluster, error) {
|
||||
return s.updateKVMTags(selector, vm, tags, true)
|
||||
}
|
||||
|
||||
func (s *Service) RemoveKVMTag(selector, vm string, tags []string) (Cluster, error) {
|
||||
return s.updateKVMTags(selector, vm, tags, false)
|
||||
}
|
||||
|
||||
func (s *Service) updateKVMTags(selector, vm string, tags []string, add bool) (Cluster, error) {
|
||||
vm = strings.TrimSpace(vm)
|
||||
if vm == "" {
|
||||
return Cluster{}, errors.New("vm name is required")
|
||||
}
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
if c.KVMTags == nil {
|
||||
c.KVMTags = map[string][]string{}
|
||||
}
|
||||
current := make(map[string]struct{}, len(c.KVMTags[vm]))
|
||||
for _, t := range normalizeTagList(c.KVMTags[vm]) {
|
||||
current[t] = struct{}{}
|
||||
}
|
||||
changes := normalizeTagList(tags)
|
||||
if len(changes) == 0 {
|
||||
return Cluster{}, errors.New("at least one non-empty tag is required")
|
||||
}
|
||||
if add {
|
||||
for _, t := range changes {
|
||||
current[t] = struct{}{}
|
||||
}
|
||||
} else {
|
||||
for _, t := range changes {
|
||||
delete(current, t)
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(current))
|
||||
for t := range current {
|
||||
out = append(out, t)
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) == 0 {
|
||||
delete(c.KVMTags, vm)
|
||||
} else {
|
||||
c.KVMTags[vm] = out
|
||||
}
|
||||
c.KVMTags = normalizeVMTagMap(c.KVMTags)
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("kvm.tags", c.Name, vm+"="+strings.Join(out, ","))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListKVMTags(selector, vm string) (map[string][]string, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make(map[string][]string, len(c.KVMTags))
|
||||
for k, v := range c.KVMTags {
|
||||
if vm != "" && !strings.EqualFold(strings.TrimSpace(vm), k) {
|
||||
continue
|
||||
}
|
||||
out[k] = append([]string(nil), normalizeTagList(v)...)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
// agentClient wraps an *http.Client pointed at the correct base URL for a
|
||||
// cluster's agent (direct or SSH-tunneled).
|
||||
type agentClient struct {
|
||||
http *http.Client
|
||||
target string
|
||||
close func()
|
||||
}
|
||||
|
||||
func (a *agentClient) Close() {
|
||||
if a != nil && a.close != nil {
|
||||
a.close()
|
||||
}
|
||||
}
|
||||
|
||||
// newAgentClient builds the right HTTP client for reaching a cluster's agent.
|
||||
//
|
||||
// For TransportDirect it returns a plain client talking to cluster.Host:port.
|
||||
//
|
||||
// For TransportIPFabric it reuses a cached ssh.Client from the service pool
|
||||
// and returns a client whose Transport routes every TCP connection through
|
||||
// ssh.Client.Dial to 127.0.0.1:port. The SSH connection stays pooled after
|
||||
// Close() — only the HTTP transport's idle conns are released.
|
||||
func (s *Service) newAgentClient(ctx context.Context, c Cluster, timeout time.Duration) (*agentClient, error) {
|
||||
if c.Agent.Port == 0 {
|
||||
return nil, errors.New("agent port is not set")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = 8 * time.Second
|
||||
}
|
||||
|
||||
scheme := agentScheme(c)
|
||||
tlsCfg, err := agentTLSConfig(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch normalizeTransport(c.Transport) {
|
||||
case TransportIPFabric:
|
||||
sshClient, err := s.acquireTunnelClient(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: tlsCfg,
|
||||
DialContext: func(dctx context.Context, network, _ string) (net.Conn, error) {
|
||||
addr := net.JoinHostPort("127.0.0.1", strconv.Itoa(c.Agent.Port))
|
||||
return sshDialWithContext(dctx, sshClient, network, addr)
|
||||
},
|
||||
DisableKeepAlives: true,
|
||||
IdleConnTimeout: 30 * time.Second,
|
||||
ResponseHeaderTimeout: timeout,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
}
|
||||
httpClient := &http.Client{
|
||||
Timeout: timeout,
|
||||
Transport: tr,
|
||||
}
|
||||
return &agentClient{
|
||||
http: httpClient,
|
||||
target: scheme + "://127.0.0.1:" + strconv.Itoa(c.Agent.Port),
|
||||
close: func() {
|
||||
tr.CloseIdleConnections()
|
||||
},
|
||||
}, nil
|
||||
|
||||
default:
|
||||
tr := &http.Transport{
|
||||
TLSClientConfig: tlsCfg,
|
||||
ResponseHeaderTimeout: timeout,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
}
|
||||
httpClient := &http.Client{Timeout: timeout, Transport: tr}
|
||||
return &agentClient{
|
||||
http: httpClient,
|
||||
target: scheme + "://" + net.JoinHostPort(c.Host, strconv.Itoa(c.Agent.Port)),
|
||||
close: func() {
|
||||
tr.CloseIdleConnections()
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func agentScheme(c Cluster) string {
|
||||
if c.Agent.TLSEnabled {
|
||||
return "https"
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
|
||||
func agentTLSConfig(c Cluster) (*tls.Config, error) {
|
||||
if !c.Agent.TLSEnabled {
|
||||
return nil, nil
|
||||
}
|
||||
fp := strings.ToLower(strings.TrimSpace(c.Agent.TLSFingerprint))
|
||||
if fp == "" {
|
||||
return nil, errors.New("agent TLS is enabled but certificate fingerprint is missing")
|
||||
}
|
||||
fp = strings.ReplaceAll(fp, ":", "")
|
||||
want, err := hex.DecodeString(fp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid agent TLS fingerprint: %w", err)
|
||||
}
|
||||
return &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
InsecureSkipVerify: true, // verified via explicit fingerprint pinning below
|
||||
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
|
||||
if len(rawCerts) == 0 {
|
||||
return errors.New("agent TLS: peer certificate is missing")
|
||||
}
|
||||
sum := sha256.Sum256(rawCerts[0])
|
||||
if len(want) != len(sum) {
|
||||
return errors.New("agent TLS: fingerprint length mismatch")
|
||||
}
|
||||
if !hmacEqual(sum[:], want) {
|
||||
return errors.New("agent TLS: fingerprint mismatch")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func hmacEqual(a, b []byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
var v byte
|
||||
for i := 0; i < len(a); i++ {
|
||||
v |= a[i] ^ b[i]
|
||||
}
|
||||
return v == 0
|
||||
}
|
||||
|
||||
// acquireTunnelClient returns a pooled ssh.Client for the cluster, creating
|
||||
// one if necessary. Credential changes invalidate the cached entry via the
|
||||
// fingerprint field. Dead clients are evicted lazily: when a DialContext
|
||||
// through a stale client fails, the caller invokes CloseTunnelClient and the
|
||||
// next acquire re-dials.
|
||||
func (s *Service) acquireTunnelClient(ctx context.Context, c Cluster) (*ssh.Client, error) {
|
||||
fp := credentialFingerprint(c)
|
||||
|
||||
s.tunnelMu.Lock()
|
||||
entry, ok := s.tunnelPool[c.ID]
|
||||
if ok && entry.fp != fp {
|
||||
_ = entry.client.Close()
|
||||
delete(s.tunnelPool, c.ID)
|
||||
entry = nil
|
||||
ok = false
|
||||
}
|
||||
if ok {
|
||||
s.tunnelMu.Unlock()
|
||||
return entry.client, nil
|
||||
}
|
||||
s.tunnelMu.Unlock()
|
||||
|
||||
client, err := s.dialSSH(ctx, c, "", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.tunnelMu.Lock()
|
||||
if existing, ok := s.tunnelPool[c.ID]; ok && existing.fp == fp {
|
||||
// Another goroutine won the race; drop ours.
|
||||
s.tunnelMu.Unlock()
|
||||
_ = client.Close()
|
||||
return existing.client, nil
|
||||
}
|
||||
s.tunnelPool[c.ID] = &tunneledSSH{
|
||||
client: client,
|
||||
fp: fp,
|
||||
}
|
||||
s.tunnelMu.Unlock()
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// CloseTunnelClient drops a pooled SSH tunnel for a cluster. Safe to call if
|
||||
// no entry exists.
|
||||
func (s *Service) CloseTunnelClient(clusterID string) {
|
||||
s.tunnelMu.Lock()
|
||||
entry, ok := s.tunnelPool[clusterID]
|
||||
if ok {
|
||||
delete(s.tunnelPool, clusterID)
|
||||
}
|
||||
s.tunnelMu.Unlock()
|
||||
if ok && entry != nil && entry.client != nil {
|
||||
_ = entry.client.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// CloseAllTunnelClients tears down every pooled SSH tunnel.
|
||||
func (s *Service) CloseAllTunnelClients() {
|
||||
s.tunnelMu.Lock()
|
||||
pool := s.tunnelPool
|
||||
s.tunnelPool = make(map[string]*tunneledSSH)
|
||||
s.tunnelMu.Unlock()
|
||||
for _, e := range pool {
|
||||
if e != nil && e.client != nil {
|
||||
_ = e.client.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// credentialFingerprint returns a short hash over the fields that affect how
|
||||
// we'd reconnect. If any of these change we must not reuse a cached client.
|
||||
func credentialFingerprint(c Cluster) string {
|
||||
h := sha256.New()
|
||||
h.Write([]byte(c.Host))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(strconv.Itoa(c.Port)))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.User))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.AuthMethod))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.Password))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.KeyPath))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.KeyPassphrase))
|
||||
h.Write([]byte{'|'})
|
||||
h.Write([]byte(c.KeyPassphraseFile))
|
||||
return hex.EncodeToString(h.Sum(nil)[:8])
|
||||
}
|
||||
|
||||
// sshDialWithContext wraps ssh.Client.Dial so it respects ctx cancellation.
|
||||
// ssh.Client has no context-aware dial, so we fall back to a watcher goroutine
|
||||
// that closes the connection if ctx fires before the dial returns.
|
||||
func sshDialWithContext(ctx context.Context, client *ssh.Client, network, addr string) (net.Conn, error) {
|
||||
type result struct {
|
||||
conn net.Conn
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
conn, err := client.Dial(network, addr)
|
||||
ch <- result{conn: conn, err: err}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
go func() {
|
||||
r := <-ch
|
||||
if r.conn != nil {
|
||||
_ = r.conn.Close()
|
||||
}
|
||||
}()
|
||||
return nil, ctx.Err()
|
||||
case r := <-ch:
|
||||
return r.conn, r.err
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"pxmon/internal/agent"
|
||||
"pxmon/internal/history"
|
||||
)
|
||||
|
||||
// UsageRange is a façade over history.RangeShortcut so callers outside this
|
||||
// package don't need to import both.
|
||||
type UsageRange = history.RangeShortcut
|
||||
|
||||
// UsageSnapshot bundles everything the "usage" page shows for a cluster at a
|
||||
// single point in time.
|
||||
type UsageSnapshot struct {
|
||||
ClusterID string `json:"cluster_id"`
|
||||
ClusterName string `json:"cluster_name"`
|
||||
Range string `json:"range"`
|
||||
GeneratedAt time.Time `json:"generated_at"`
|
||||
Live agent.StatsResponse `json:"live"`
|
||||
Top agent.TopResponse `json:"top"`
|
||||
DU agent.DUResponse `json:"du"`
|
||||
NodeSeries []history.NodeSamplePoint `json:"series,omitempty"`
|
||||
P95TotalMbps float64 `json:"p95_total_mbps"`
|
||||
MaxTotalMbps float64 `json:"max_total_mbps"`
|
||||
AvgTotalMbps float64 `json:"avg_total_mbps"`
|
||||
TopIfaceName string `json:"top_iface_name,omitempty"`
|
||||
TopIfaceMbps float64 `json:"top_iface_mbps,omitempty"`
|
||||
DUError string `json:"du_error,omitempty"`
|
||||
TopError string `json:"top_error,omitempty"`
|
||||
HistoryError string `json:"history_error,omitempty"`
|
||||
}
|
||||
|
||||
// AgentTopProcesses calls /api/v1/top on the selected cluster's agent.
|
||||
func (s *Service) AgentTopProcesses(ctx context.Context, selector string, sampleWindow time.Duration, limit int) (agent.TopResponse, error) {
|
||||
cluster, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
if !cluster.Agent.Installed {
|
||||
return agent.TopResponse{}, errors.New("agent is not installed on this cluster")
|
||||
}
|
||||
|
||||
ac, err := s.newAgentClient(ctx, cluster, 15*time.Second)
|
||||
if err != nil {
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
defer ac.Close()
|
||||
|
||||
url := ac.target + "/api/v1/top"
|
||||
q := ""
|
||||
if sampleWindow > 0 {
|
||||
q += "sample_ms=" + strconv.FormatInt(sampleWindow.Milliseconds(), 10)
|
||||
}
|
||||
if limit > 0 {
|
||||
if q != "" {
|
||||
q += "&"
|
||||
}
|
||||
q += "limit=" + strconv.Itoa(limit)
|
||||
}
|
||||
if q != "" {
|
||||
url += "?" + q
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
applyAgentRequestAuth(req, cluster)
|
||||
|
||||
resp, err := ac.http.Do(req)
|
||||
if err != nil {
|
||||
if normalizeTransport(cluster.Transport) == TransportIPFabric {
|
||||
s.CloseTunnelClient(cluster.ID)
|
||||
}
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
msg := string(body)
|
||||
if len(msg) > 200 {
|
||||
msg = msg[:200]
|
||||
}
|
||||
return agent.TopResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
|
||||
}
|
||||
|
||||
var out agent.TopResponse
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return agent.TopResponse{}, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AgentDirSizes calls /api/v1/du on the selected cluster's agent.
|
||||
func (s *Service) AgentDirSizes(ctx context.Context, selector, path string, limit int, timeout time.Duration) (agent.DUResponse, error) {
|
||||
cluster, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
if !cluster.Agent.Installed {
|
||||
return agent.DUResponse{}, errors.New("agent is not installed on this cluster")
|
||||
}
|
||||
|
||||
httpTimeout := timeout + 10*time.Second
|
||||
if httpTimeout < 20*time.Second {
|
||||
httpTimeout = 20 * time.Second
|
||||
}
|
||||
ac, err := s.newAgentClient(ctx, cluster, httpTimeout)
|
||||
if err != nil {
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
defer ac.Close()
|
||||
|
||||
url := ac.target + "/api/v1/du"
|
||||
q := ""
|
||||
if path != "" {
|
||||
q += "path=" + path
|
||||
}
|
||||
if limit > 0 {
|
||||
if q != "" {
|
||||
q += "&"
|
||||
}
|
||||
q += "limit=" + strconv.Itoa(limit)
|
||||
}
|
||||
if timeout > 0 {
|
||||
if q != "" {
|
||||
q += "&"
|
||||
}
|
||||
q += "timeout_ms=" + strconv.FormatInt(timeout.Milliseconds(), 10)
|
||||
}
|
||||
if q != "" {
|
||||
url += "?" + q
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
applyAgentRequestAuth(req, cluster)
|
||||
|
||||
resp, err := ac.http.Do(req)
|
||||
if err != nil {
|
||||
if normalizeTransport(cluster.Transport) == TransportIPFabric {
|
||||
s.CloseTunnelClient(cluster.ID)
|
||||
}
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
msg := string(body)
|
||||
if len(msg) > 200 {
|
||||
msg = msg[:200]
|
||||
}
|
||||
return agent.DUResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
|
||||
}
|
||||
|
||||
var out agent.DUResponse
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return agent.DUResponse{}, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CollectUsageSnapshot gathers every data source needed by the usage page:
|
||||
// live stats, top processes, top folders, and historical aggregate for P95.
|
||||
// Errors in individual components are recorded on the snapshot instead of
|
||||
// aborting the whole call — the page degrades gracefully.
|
||||
func (s *Service) CollectUsageSnapshot(ctx context.Context, selector string, rng UsageRange, duPath string) (UsageSnapshot, error) {
|
||||
cluster, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return UsageSnapshot{}, err
|
||||
}
|
||||
|
||||
snap := UsageSnapshot{
|
||||
ClusterID: cluster.ID,
|
||||
ClusterName: cluster.Name,
|
||||
Range: string(rng),
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
if live, err := s.AgentStatsTyped(ctx, selector); err != nil {
|
||||
return snap, fmt.Errorf("live stats: %w", err)
|
||||
} else {
|
||||
snap.Live = live
|
||||
}
|
||||
|
||||
if top, err := s.AgentTopProcesses(ctx, selector, 300*time.Millisecond, 15); err != nil {
|
||||
snap.TopError = err.Error()
|
||||
} else {
|
||||
snap.Top = top
|
||||
}
|
||||
|
||||
if duPath == "" {
|
||||
duPath = "/"
|
||||
}
|
||||
if du, err := s.AgentDirSizes(ctx, selector, duPath, 12, 12*time.Second); err != nil {
|
||||
snap.DUError = err.Error()
|
||||
} else {
|
||||
snap.DU = du
|
||||
}
|
||||
|
||||
// Network history + P95 aggregation.
|
||||
if s.networkStore == nil {
|
||||
snap.HistoryError = "history store not configured"
|
||||
} else {
|
||||
since := history.RangeShortcut(rng).Since(time.Now())
|
||||
if rng == "" || rng == history.RangeLive {
|
||||
since = time.Now().Add(-5 * time.Minute)
|
||||
}
|
||||
snapshots, err := s.networkStore.Load(cluster.ID, since)
|
||||
if err != nil {
|
||||
snap.HistoryError = err.Error()
|
||||
} else {
|
||||
// Pick the physical uplink once and use it consistently for
|
||||
// the series, P95, and the "top iface" display so they all
|
||||
// describe the same thing.
|
||||
primary := history.PrimaryInterface(snapshots)
|
||||
series := history.AggregateNodeSeries(snapshots, primary)
|
||||
snap.NodeSeries = series
|
||||
snap.P95TotalMbps = history.PercentileMbps(series, 95)
|
||||
if len(series) > 0 {
|
||||
maxV := 0.0
|
||||
sum := 0.0
|
||||
for _, p := range series {
|
||||
if p.TotalMbps > maxV {
|
||||
maxV = p.TotalMbps
|
||||
}
|
||||
sum += p.TotalMbps
|
||||
}
|
||||
snap.MaxTotalMbps = maxV
|
||||
snap.AvgTotalMbps = sum / float64(len(series))
|
||||
}
|
||||
snap.TopIfaceName = primary
|
||||
snap.TopIfaceMbps = snap.AvgTotalMbps
|
||||
}
|
||||
}
|
||||
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// RenderUsageChartPNG is a thin helper that reads a usage snapshot's series
|
||||
// and delegates to history.RenderNodeNetworkPNG.
|
||||
func RenderUsageChartPNG(snap UsageSnapshot, title string) ([]byte, error) {
|
||||
if title == "" {
|
||||
title = fmt.Sprintf("%s — node network usage (%s)", snap.ClusterName, snap.Range)
|
||||
}
|
||||
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps",
|
||||
snap.P95TotalMbps, snap.MaxTotalMbps, snap.AvgTotalMbps)
|
||||
return history.RenderNodeNetworkPNG(snap.NodeSeries, history.ChartOptions{
|
||||
Title: title,
|
||||
Subtitle: subtitle,
|
||||
Percentile: 95,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type VMStateSummary struct {
|
||||
Total int `json:"total"`
|
||||
Running int `json:"running"`
|
||||
ShutOff int `json:"shut_off"`
|
||||
Paused int `json:"paused"`
|
||||
Others int `json:"others"`
|
||||
ShutOffNames []string `json:"shut_off_names,omitempty"`
|
||||
PausedNames []string `json:"paused_names,omitempty"`
|
||||
OtherNames []string `json:"other_names,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
SampledAt time.Time `json:"sampled_at"`
|
||||
}
|
||||
|
||||
func (s *Service) GetVMAlertPolicy(selector string) (VMAlertPolicy, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return VMAlertPolicy{}, err
|
||||
}
|
||||
return ensureVMAlertPolicy(c.VMAlerts), nil
|
||||
}
|
||||
|
||||
func (s *Service) SetVMAlertPolicy(selector string, p VMAlertPolicy) (Cluster, error) {
|
||||
reg, err := s.store.Load()
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c, idx, err := findCluster(reg, selector)
|
||||
if err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
c.VMAlerts = ensureVMAlertPolicy(p)
|
||||
c.UpdatedAt = s.now().UTC()
|
||||
reg.Clusters[idx] = c
|
||||
if err := s.store.Save(reg); err != nil {
|
||||
return Cluster{}, err
|
||||
}
|
||||
_ = s.AppendChange("vm.alerts", c.Name, fmt.Sprintf("enabled=%t warn_on_shutoff=%t min_running=%d", c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Service) CheckVMAlerts(ctx context.Context, selector string) (VMStateSummary, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return VMStateSummary{}, err
|
||||
}
|
||||
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
|
||||
if err != nil {
|
||||
return VMStateSummary{}, err
|
||||
}
|
||||
states := parseVirshListStates(out)
|
||||
summary := VMStateSummary{SampledAt: s.now().UTC(), Total: len(states)}
|
||||
for name, st := range states {
|
||||
n := strings.ToLower(strings.TrimSpace(st))
|
||||
switch {
|
||||
case n == "running":
|
||||
summary.Running++
|
||||
case n == "shut off":
|
||||
summary.ShutOff++
|
||||
summary.ShutOffNames = append(summary.ShutOffNames, name)
|
||||
case n == "paused":
|
||||
summary.Paused++
|
||||
summary.PausedNames = append(summary.PausedNames, name)
|
||||
default:
|
||||
summary.Others++
|
||||
summary.OtherNames = append(summary.OtherNames, name)
|
||||
}
|
||||
}
|
||||
sort.Strings(summary.ShutOffNames)
|
||||
sort.Strings(summary.PausedNames)
|
||||
sort.Strings(summary.OtherNames)
|
||||
p := ensureVMAlertPolicy(c.VMAlerts)
|
||||
if p.Enabled {
|
||||
if p.WarnOnShutoff && summary.ShutOff > 0 {
|
||||
summary.Warnings = append(summary.Warnings, fmt.Sprintf(
|
||||
"%d VM(s) are shut off: %s",
|
||||
summary.ShutOff,
|
||||
joinNamesLimit(summary.ShutOffNames, 12),
|
||||
))
|
||||
}
|
||||
if summary.Running < p.MinRunning {
|
||||
summary.Warnings = append(summary.Warnings, fmt.Sprintf("running VM count %d is below min_running=%d", summary.Running, p.MinRunning))
|
||||
}
|
||||
}
|
||||
sort.Strings(summary.Warnings)
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (s *Service) ListVMStates(ctx context.Context, selector string) (map[string]string, error) {
|
||||
c, err := s.Get(selector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseVirshListStates(out), nil
|
||||
}
|
||||
|
||||
func parseVirshListStates(raw string) map[string]string {
|
||||
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
||||
out := make(map[string]string)
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "Id") || strings.HasPrefix(line, "-") {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) < 3 {
|
||||
continue
|
||||
}
|
||||
name := fields[1]
|
||||
state := strings.Join(fields[2:], " ")
|
||||
out[name] = state
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func joinNamesLimit(items []string, limit int) string {
|
||||
if len(items) == 0 {
|
||||
return "-"
|
||||
}
|
||||
if limit <= 0 || len(items) <= limit {
|
||||
return strings.Join(items, ", ")
|
||||
}
|
||||
return strings.Join(items[:limit], ", ") + fmt.Sprintf(" (+%d more)", len(items)-limit)
|
||||
}
|
||||
Reference in New Issue
Block a user