519 lines
15 KiB
Go
519 lines
15 KiB
Go
package cluster
|
|
|
|
import (
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const currentVersion = 8
|
|
|
|
// AuthMethod describes how SSH authentication is performed.
|
|
type AuthMethod string
|
|
|
|
const (
|
|
AuthMethodPassword AuthMethod = "password"
|
|
AuthMethodKey AuthMethod = "key"
|
|
)
|
|
|
|
// TransportMode selects how pxmon reaches the agent HTTP API on a node.
|
|
type TransportMode string
|
|
|
|
const (
|
|
// TransportDirect is the default: pxmon dials the agent's listen
|
|
// address over plain TCP from the local machine.
|
|
TransportDirect TransportMode = "direct"
|
|
// TransportIPFabric tunnels the agent HTTP call through the existing SSH
|
|
// connection. Meant for nodes on ipfabric-style networking where the node
|
|
// has no default outbound route and we must not touch its network config.
|
|
// The agent is expected to bind to 127.0.0.1 on the node.
|
|
TransportIPFabric TransportMode = "ipfabric"
|
|
)
|
|
|
|
func normalizeTransport(t TransportMode) TransportMode {
|
|
switch strings.ToLower(strings.TrimSpace(string(t))) {
|
|
case "ipfabric", "ip-fabric", "ip_fabric":
|
|
return TransportIPFabric
|
|
case "", "direct":
|
|
return TransportDirect
|
|
default:
|
|
return TransportDirect
|
|
}
|
|
}
|
|
|
|
// Registry is the local inventory of managed clusters (nodes).
|
|
type Registry struct {
|
|
Version int `json:"version"`
|
|
ActiveClusterID string `json:"active_cluster_id,omitempty"`
|
|
Telegram Telegram `json:"telegram,omitempty"`
|
|
Locker Locker `json:"locker,omitempty"`
|
|
Backups BackupConfig `json:"backups,omitempty"`
|
|
Clusters []Cluster `json:"clusters"`
|
|
}
|
|
|
|
type BackupConfig struct {
|
|
Targets []BackupTarget `json:"targets,omitempty"`
|
|
Plans []BackupPlan `json:"plans,omitempty"`
|
|
}
|
|
|
|
type BackupTarget struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Type string `json:"type"` // sftp|s3
|
|
Enabled bool `json:"enabled"`
|
|
|
|
SFTPHost string `json:"sftp_host,omitempty"`
|
|
SFTPPort int `json:"sftp_port,omitempty"`
|
|
SFTPUser string `json:"sftp_user,omitempty"`
|
|
SFTPPassword string `json:"sftp_password,omitempty"`
|
|
SFTPKeyPath string `json:"sftp_key_path,omitempty"`
|
|
SFTPBasePath string `json:"sftp_base_path,omitempty"`
|
|
|
|
S3Endpoint string `json:"s3_endpoint,omitempty"`
|
|
S3Region string `json:"s3_region,omitempty"`
|
|
S3Bucket string `json:"s3_bucket,omitempty"`
|
|
S3Prefix string `json:"s3_prefix,omitempty"`
|
|
S3AccessKey string `json:"s3_access_key,omitempty"`
|
|
S3SecretKey string `json:"s3_secret_key,omitempty"`
|
|
S3UseSSL bool `json:"s3_use_ssl,omitempty"`
|
|
S3PathStyle bool `json:"s3_path_style,omitempty"`
|
|
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
type BackupPlan struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Cluster string `json:"cluster,omitempty"`
|
|
TargetID string `json:"target_id"`
|
|
Paths []string `json:"paths"`
|
|
Every string `json:"every,omitempty"`
|
|
Enabled bool `json:"enabled"`
|
|
RetainDays int `json:"retain_days,omitempty"`
|
|
Compress bool `json:"compress"`
|
|
LastRunAt time.Time `json:"last_run_at,omitempty"`
|
|
LastStatus string `json:"last_status,omitempty"`
|
|
LastArchive string `json:"last_archive,omitempty"`
|
|
LastError string `json:"last_error,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
// Telegram stores Telegram bot integration settings.
|
|
type Telegram struct {
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
Token string `json:"token,omitempty"`
|
|
AllowedUserIDs []int64 `json:"allowed_user_ids,omitempty"`
|
|
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
|
}
|
|
|
|
// Locker stores global UI/CLI lock settings.
|
|
type Locker struct {
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
PasswordHash string `json:"password_hash,omitempty"`
|
|
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
|
}
|
|
|
|
// Cluster describes one managed node that we can reach over SSH.
|
|
type Cluster struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name"`
|
|
Host string `json:"host"`
|
|
Port int `json:"port"`
|
|
User string `json:"user"`
|
|
Transport TransportMode `json:"transport,omitempty"`
|
|
AuthMethod AuthMethod `json:"auth_method"`
|
|
Password string `json:"password,omitempty"`
|
|
KeyPath string `json:"key_path,omitempty"`
|
|
KeyPassphrase string `json:"key_passphrase,omitempty"`
|
|
KeyPassphraseFile string `json:"key_passphrase_file,omitempty"`
|
|
InsecureHostKey bool `json:"insecure_host_key,omitempty"`
|
|
Alerts AlertPolicy `json:"alerts"`
|
|
VMAlerts VMAlertPolicy `json:"vm_alerts,omitempty"`
|
|
AlertRouting AlertRoutingPolicy `json:"alert_routing,omitempty"`
|
|
RepoTunnel RepoTunnelState `json:"repo_tunnel,omitempty"`
|
|
RunbookTrigger RunbookTrigger `json:"runbook_trigger,omitempty"`
|
|
Drift DriftControl `json:"drift,omitempty"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
KVMTags map[string][]string `json:"kvm_tags,omitempty"`
|
|
Agent AgentInstall `json:"agent,omitempty"`
|
|
Software SoftwareInfo `json:"software,omitempty"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
UpdatedAt time.Time `json:"updated_at"`
|
|
}
|
|
|
|
// AlertPolicy defines warning thresholds used by CLI monitor mode.
|
|
type AlertPolicy struct {
|
|
CPUWarnPercent float64 `json:"cpu_warn_percent"`
|
|
RAMWarnPercent float64 `json:"ram_warn_percent"`
|
|
SwapWarnPercent float64 `json:"swap_warn_percent"`
|
|
DiskWarnPercent float64 `json:"disk_warn_percent"`
|
|
NetWarnMbps float64 `json:"net_warn_mbps"`
|
|
NetSustainEnabled bool `json:"net_sustain_enabled,omitempty"`
|
|
NetSustainIface string `json:"net_sustain_iface,omitempty"`
|
|
NetSustainInclude []string `json:"net_sustain_include,omitempty"`
|
|
NetSustainExclude []string `json:"net_sustain_exclude,omitempty"`
|
|
NetSustainMbps float64 `json:"net_sustain_mbps,omitempty"`
|
|
NetSustainMinutes int `json:"net_sustain_minutes,omitempty"`
|
|
NetSustainCooldownMins int `json:"net_sustain_cooldown_mins,omitempty"`
|
|
}
|
|
|
|
// VMAlertPolicy controls KVM VM-state alerting per cluster.
|
|
type VMAlertPolicy struct {
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
WarnOnShutoff bool `json:"warn_on_shutoff,omitempty"`
|
|
MinRunning int `json:"min_running,omitempty"`
|
|
}
|
|
|
|
// AlertRoutingPolicy controls delivery behavior for alerts (e.g. Telegram).
|
|
type AlertRoutingPolicy struct {
|
|
CriticalImmediate bool `json:"critical_immediate,omitempty"`
|
|
WarningBatchMins int `json:"warning_batch_mins,omitempty"`
|
|
}
|
|
|
|
// RunbookTrigger controls automatic runbook execution on selected events.
|
|
type RunbookTrigger struct {
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
OnVMShutoff bool `json:"on_vm_shutoff,omitempty"`
|
|
RunbookID string `json:"runbook_id,omitempty"`
|
|
CooldownMins int `json:"cooldown_mins,omitempty"`
|
|
LastTriggered time.Time `json:"last_triggered,omitempty"`
|
|
}
|
|
|
|
// DriftBaseline stores expected values for drift comparisons.
|
|
type DriftBaseline struct {
|
|
Enabled bool `json:"enabled,omitempty"`
|
|
SetAt time.Time `json:"set_at,omitempty"`
|
|
AgentVersion string `json:"agent_version,omitempty"`
|
|
Software string `json:"software,omitempty"`
|
|
}
|
|
|
|
// DriftControl stores baseline and per-issue acknowledgement windows.
|
|
type DriftControl struct {
|
|
Baseline DriftBaseline `json:"baseline,omitempty"`
|
|
AckUntil map[string]time.Time `json:"ack_until,omitempty"`
|
|
}
|
|
|
|
// AgentInstall describes remote pxmon-agent installation details.
|
|
type AgentInstall struct {
|
|
Installed bool `json:"installed"`
|
|
Version string `json:"version,omitempty"`
|
|
RemoteBinary string `json:"remote_binary,omitempty"`
|
|
RemoteConfig string `json:"remote_config,omitempty"`
|
|
RemoteLog string `json:"remote_log,omitempty"`
|
|
RemotePIDFile string `json:"remote_pid_file,omitempty"`
|
|
ListenAddress string `json:"listen_address,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
Token string `json:"token,omitempty"`
|
|
RequestSecret string `json:"request_secret,omitempty"`
|
|
TLSEnabled bool `json:"tls_enabled,omitempty"`
|
|
TLSCertPath string `json:"tls_cert_path,omitempty"`
|
|
TLSKeyPath string `json:"tls_key_path,omitempty"`
|
|
TLSFingerprint string `json:"tls_fingerprint,omitempty"`
|
|
LastBootstrapAt time.Time `json:"last_bootstrap_at,omitempty"`
|
|
}
|
|
|
|
// SoftwareInfo describes discovered software/plugins on a node.
|
|
type SoftwareInfo struct {
|
|
DetectedAt time.Time `json:"detected_at,omitempty"`
|
|
Bird bool `json:"bird,omitempty"`
|
|
FRR bool `json:"frr,omitempty"`
|
|
KVM bool `json:"kvm,omitempty"`
|
|
LXC bool `json:"lxc,omitempty"`
|
|
LXD bool `json:"lxd,omitempty"`
|
|
Versions map[string]string `json:"versions,omitempty"`
|
|
}
|
|
|
|
func (s SoftwareInfo) SupportedList() []string {
|
|
out := make([]string, 0, 5)
|
|
if s.Bird {
|
|
out = append(out, "bird")
|
|
}
|
|
if s.FRR {
|
|
out = append(out, "frr")
|
|
}
|
|
if s.KVM {
|
|
out = append(out, "kvm")
|
|
}
|
|
if s.LXC {
|
|
out = append(out, "lxc")
|
|
}
|
|
if s.LXD {
|
|
out = append(out, "lxd")
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func (s SoftwareInfo) Summary() string {
|
|
list := s.SupportedList()
|
|
if len(list) == 0 {
|
|
if !s.DetectedAt.IsZero() {
|
|
return "none"
|
|
}
|
|
return "-"
|
|
}
|
|
return strings.Join(list, ",")
|
|
}
|
|
|
|
func newRegistry() Registry {
|
|
return Registry{
|
|
Version: currentVersion,
|
|
Backups: normalizeBackupConfig(BackupConfig{}),
|
|
Clusters: []Cluster{},
|
|
}
|
|
}
|
|
|
|
func defaultAlertPolicy() AlertPolicy {
|
|
return AlertPolicy{
|
|
CPUWarnPercent: 85,
|
|
RAMWarnPercent: 90,
|
|
SwapWarnPercent: 80,
|
|
DiskWarnPercent: 90,
|
|
NetWarnMbps: 300,
|
|
}
|
|
}
|
|
|
|
func defaultVMAlertPolicy() VMAlertPolicy {
|
|
return VMAlertPolicy{
|
|
Enabled: false,
|
|
WarnOnShutoff: true,
|
|
MinRunning: 1,
|
|
}
|
|
}
|
|
|
|
func defaultAlertRoutingPolicy() AlertRoutingPolicy {
|
|
return AlertRoutingPolicy{
|
|
CriticalImmediate: true,
|
|
WarningBatchMins: 5,
|
|
}
|
|
}
|
|
|
|
func defaultRunbookTrigger() RunbookTrigger {
|
|
return RunbookTrigger{
|
|
Enabled: false,
|
|
OnVMShutoff: true,
|
|
CooldownMins: 30,
|
|
}
|
|
}
|
|
|
|
func ensureAlertPolicy(p AlertPolicy) AlertPolicy {
|
|
d := defaultAlertPolicy()
|
|
if p.CPUWarnPercent <= 0 {
|
|
p.CPUWarnPercent = d.CPUWarnPercent
|
|
}
|
|
if p.RAMWarnPercent <= 0 {
|
|
p.RAMWarnPercent = d.RAMWarnPercent
|
|
}
|
|
if p.SwapWarnPercent <= 0 {
|
|
p.SwapWarnPercent = d.SwapWarnPercent
|
|
}
|
|
if p.DiskWarnPercent <= 0 {
|
|
p.DiskWarnPercent = d.DiskWarnPercent
|
|
}
|
|
if p.NetWarnMbps <= 0 {
|
|
p.NetWarnMbps = d.NetWarnMbps
|
|
}
|
|
if p.NetSustainEnabled {
|
|
if p.NetSustainMbps <= 0 {
|
|
p.NetSustainMbps = d.NetWarnMbps
|
|
}
|
|
if p.NetSustainMinutes <= 0 {
|
|
p.NetSustainMinutes = 60
|
|
}
|
|
if p.NetSustainCooldownMins <= 0 {
|
|
p.NetSustainCooldownMins = 30
|
|
}
|
|
}
|
|
return p
|
|
}
|
|
|
|
func ensureVMAlertPolicy(p VMAlertPolicy) VMAlertPolicy {
|
|
d := defaultVMAlertPolicy()
|
|
wasZero := p == (VMAlertPolicy{})
|
|
if p.MinRunning <= 0 {
|
|
p.MinRunning = d.MinRunning
|
|
}
|
|
if !p.WarnOnShutoff {
|
|
// Keep explicit false if user set it, but default to true for zero-value
|
|
// policy loaded from old configs.
|
|
if wasZero {
|
|
p.WarnOnShutoff = d.WarnOnShutoff
|
|
}
|
|
}
|
|
return p
|
|
}
|
|
|
|
func ensureAlertRoutingPolicy(p AlertRoutingPolicy) AlertRoutingPolicy {
|
|
d := defaultAlertRoutingPolicy()
|
|
if p.WarningBatchMins <= 0 {
|
|
p.WarningBatchMins = d.WarningBatchMins
|
|
}
|
|
// default true when unset
|
|
if !p.CriticalImmediate && p == (AlertRoutingPolicy{}) {
|
|
p.CriticalImmediate = d.CriticalImmediate
|
|
}
|
|
return p
|
|
}
|
|
|
|
func ensureRunbookTrigger(p RunbookTrigger) RunbookTrigger {
|
|
d := defaultRunbookTrigger()
|
|
if p.CooldownMins <= 0 {
|
|
p.CooldownMins = d.CooldownMins
|
|
}
|
|
if !p.OnVMShutoff && p == (RunbookTrigger{}) {
|
|
p.OnVMShutoff = d.OnVMShutoff
|
|
}
|
|
p.RunbookID = strings.TrimSpace(p.RunbookID)
|
|
return p
|
|
}
|
|
|
|
func normalizeBackupConfig(cfg BackupConfig) BackupConfig {
|
|
if cfg.Targets == nil {
|
|
cfg.Targets = []BackupTarget{}
|
|
}
|
|
if cfg.Plans == nil {
|
|
cfg.Plans = []BackupPlan{}
|
|
}
|
|
for i := range cfg.Targets {
|
|
t := &cfg.Targets[i]
|
|
t.ID = strings.TrimSpace(t.ID)
|
|
t.Name = strings.TrimSpace(t.Name)
|
|
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
|
|
if t.SFTPPort <= 0 {
|
|
t.SFTPPort = 22
|
|
}
|
|
if t.ID == "" {
|
|
t.ID = newClusterID()
|
|
}
|
|
if t.Name == "" {
|
|
t.Name = t.ID
|
|
}
|
|
if t.Type != "sftp" && t.Type != "s3" {
|
|
t.Type = "sftp"
|
|
}
|
|
if !t.Enabled && t.CreatedAt.IsZero() {
|
|
t.Enabled = true
|
|
}
|
|
if t.CreatedAt.IsZero() {
|
|
t.CreatedAt = time.Now().UTC()
|
|
}
|
|
if t.UpdatedAt.IsZero() {
|
|
t.UpdatedAt = t.CreatedAt
|
|
}
|
|
}
|
|
for i := range cfg.Plans {
|
|
p := &cfg.Plans[i]
|
|
p.ID = strings.TrimSpace(p.ID)
|
|
p.Name = strings.TrimSpace(p.Name)
|
|
p.Cluster = strings.TrimSpace(p.Cluster)
|
|
p.TargetID = strings.TrimSpace(p.TargetID)
|
|
if p.ID == "" {
|
|
p.ID = newClusterID()
|
|
}
|
|
if p.Name == "" {
|
|
p.Name = p.ID
|
|
}
|
|
if p.Paths == nil {
|
|
p.Paths = []string{}
|
|
}
|
|
if p.Every == "" {
|
|
p.Every = "24h"
|
|
}
|
|
if p.RetainDays <= 0 {
|
|
p.RetainDays = 30
|
|
}
|
|
if !p.Compress {
|
|
p.Compress = true
|
|
}
|
|
if !p.Enabled && p.CreatedAt.IsZero() {
|
|
p.Enabled = true
|
|
}
|
|
if p.CreatedAt.IsZero() {
|
|
p.CreatedAt = time.Now().UTC()
|
|
}
|
|
if p.UpdatedAt.IsZero() {
|
|
p.UpdatedAt = p.CreatedAt
|
|
}
|
|
}
|
|
return cfg
|
|
}
|
|
|
|
func normalizeDriftControl(d DriftControl) DriftControl {
|
|
if len(d.AckUntil) == 0 {
|
|
d.AckUntil = nil
|
|
return d
|
|
}
|
|
out := make(map[string]time.Time, len(d.AckUntil))
|
|
for k, v := range d.AckUntil {
|
|
n := strings.ToLower(strings.TrimSpace(k))
|
|
if n == "" || v.IsZero() {
|
|
continue
|
|
}
|
|
out[n] = v.UTC()
|
|
}
|
|
if len(out) == 0 {
|
|
d.AckUntil = nil
|
|
} else {
|
|
d.AckUntil = out
|
|
}
|
|
d.Baseline.AgentVersion = strings.TrimSpace(d.Baseline.AgentVersion)
|
|
d.Baseline.Software = strings.TrimSpace(d.Baseline.Software)
|
|
return d
|
|
}
|
|
|
|
func normalizeTagList(tags []string) []string {
|
|
if len(tags) == 0 {
|
|
return nil
|
|
}
|
|
seen := make(map[string]struct{}, len(tags))
|
|
out := make([]string, 0, len(tags))
|
|
for _, t := range tags {
|
|
n := strings.ToLower(strings.TrimSpace(t))
|
|
if n == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[n]; ok {
|
|
continue
|
|
}
|
|
seen[n] = struct{}{}
|
|
out = append(out, n)
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|
|
|
|
func normalizeVMTagMap(m map[string][]string) map[string][]string {
|
|
if len(m) == 0 {
|
|
return nil
|
|
}
|
|
out := make(map[string][]string, len(m))
|
|
for vm, tags := range m {
|
|
vmName := strings.TrimSpace(vm)
|
|
if vmName == "" {
|
|
continue
|
|
}
|
|
norm := normalizeTagList(tags)
|
|
if len(norm) == 0 {
|
|
continue
|
|
}
|
|
out[vmName] = norm
|
|
}
|
|
if len(out) == 0 {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
func normalizeLocker(cfg Locker) Locker {
|
|
cfg.PasswordHash = strings.TrimSpace(cfg.PasswordHash)
|
|
if cfg.PasswordHash == "" {
|
|
cfg.Enabled = false
|
|
}
|
|
return cfg
|
|
}
|