chore: publish pxmon v0.2.0

This commit is contained in:
2026-06-16 21:52:10 +04:00
commit 6b703db02b
69 changed files with 25886 additions and 0 deletions
+655
View File
@@ -0,0 +1,655 @@
package cluster
import (
"context"
"fmt"
"io"
"net"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type BackupRunResult struct {
PlanID string `json:"plan_id"`
PlanName string `json:"plan_name"`
TargetID string `json:"target_id"`
TargetName string `json:"target_name"`
ArchiveName string `json:"archive_name"`
UploadedTo string `json:"uploaded_to"`
SizeBytes int64 `json:"size_bytes"`
RanAt time.Time `json:"ran_at"`
}
func (s *Service) BackupListTargets() ([]BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil
}
func (s *Service) BackupListPlans() ([]BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return nil, err
}
return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil
}
func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) {
reg, err := s.store.Load()
if err != nil {
return "", err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
var target *BackupTarget
for i := range cfg.Targets {
if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return "", fmt.Errorf("backup target not found")
}
switch target.Type {
case "sftp":
host := strings.TrimSpace(target.SFTPHost)
user := strings.TrimSpace(target.SFTPUser)
if host == "" || user == "" {
return "", fmt.Errorf("sftp target has empty host/user")
}
addr := netJoinHostPort(host, target.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(target.SFTPPassword) != "" {
auths = append(auths, ssh.Password(target.SFTPPassword))
}
if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(target.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
if _, err := c.ReadDir(base); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil
case "s3":
endpoint := strings.TrimSpace(target.S3Endpoint)
bucket := strings.TrimSpace(target.S3Bucket)
access := strings.TrimSpace(target.S3AccessKey)
secret := strings.TrimSpace(target.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(target.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if target.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: target.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
exists, err := cli.BucketExists(ctx, bucket)
if err != nil {
return "", err
}
if !exists {
return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket)
}
scheme := "https"
if !target.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket, nil
default:
return "", fmt.Errorf("unsupported target type %q", target.Type)
}
}
func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
t.ID = strings.TrimSpace(t.ID)
if t.ID == "" {
t.ID = newClusterID()
}
t.Name = strings.TrimSpace(t.Name)
if t.Name == "" {
return BackupTarget{}, fmt.Errorf("target name is required")
}
t.Type = strings.ToLower(strings.TrimSpace(t.Type))
if t.Type != "sftp" && t.Type != "s3" {
return BackupTarget{}, fmt.Errorf("target type must be sftp|s3")
}
if t.Type == "sftp" {
if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user")
}
if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" {
return BackupTarget{}, fmt.Errorf("sftp target requires password or key")
}
}
if t.Type == "s3" {
if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket")
}
if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" {
return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key")
}
}
for _, ex := range cfg.Targets {
if strings.EqualFold(ex.Name, t.Name) {
return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name)
}
}
now := s.now().UTC()
t.CreatedAt = now
t.UpdatedAt = now
if t.SFTPPort <= 0 {
t.SFTPPort = 22
}
if !t.Enabled {
t.Enabled = true
}
cfg.Targets = append(cfg.Targets, t)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.add", t.ID, t.Name)
return t, nil
}
func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) {
reg, err := s.store.Load()
if err != nil {
return BackupTarget{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, t := range cfg.Targets {
if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupTarget{}, fmt.Errorf("backup target not found")
}
removed := cfg.Targets[idx]
cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupTarget{}, err
}
_ = s.AppendChange("backup.target.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
if strings.TrimSpace(p.Name) == "" {
return BackupPlan{}, fmt.Errorf("plan name is required")
}
if strings.TrimSpace(p.TargetID) == "" {
return BackupPlan{}, fmt.Errorf("target is required")
}
if len(p.Paths) == 0 {
return BackupPlan{}, fmt.Errorf("at least one path is required")
}
targetID := ""
for _, t := range cfg.Targets {
if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) {
targetID = t.ID
break
}
}
if targetID == "" {
return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID)
}
for _, ex := range cfg.Plans {
if strings.EqualFold(ex.Name, p.Name) {
return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name)
}
}
p.ID = newClusterID()
p.TargetID = targetID
p.Paths = normalizeBackupPaths(p.Paths)
if strings.TrimSpace(p.Every) == "" {
p.Every = "24h"
}
if p.RetainDays <= 0 {
p.RetainDays = 30
}
p.Compress = true
now := s.now().UTC()
p.CreatedAt = now
p.UpdatedAt = now
if !p.Enabled {
p.Enabled = true
}
cfg.Plans = append(cfg.Plans, p)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.add", p.ID, p.Name)
return p, nil
}
func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) {
reg, err := s.store.Load()
if err != nil {
return BackupPlan{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
selector = strings.TrimSpace(selector)
idx := -1
for i, p := range cfg.Plans {
if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) {
idx = i
break
}
}
if idx < 0 {
return BackupPlan{}, fmt.Errorf("backup plan not found")
}
removed := cfg.Plans[idx]
cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...)
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupPlan{}, err
}
_ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name)
return removed, nil
}
func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) {
reg, err := s.store.Load()
if err != nil {
return BackupRunResult{}, err
}
cfg := normalizeBackupConfig(reg.Backups)
var plan *BackupPlan
for i := range cfg.Plans {
if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) {
plan = &cfg.Plans[i]
break
}
}
if plan == nil {
return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector)
}
var target *BackupTarget
for i := range cfg.Targets {
if cfg.Targets[i].ID == plan.TargetID {
target = &cfg.Targets[i]
break
}
}
if target == nil {
return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID)
}
c, err := s.Get(plan.Cluster)
if err != nil {
return BackupRunResult{}, err
}
sshClient, err := s.dialSSH(ctx, c, "", "")
if err != nil {
return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err)
}
defer sshClient.Close()
ts := s.now().UTC().Format("20060102T150405Z")
archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz"
tmpPath := filepath.Join(os.TempDir(), archiveName)
tmpFile, err := os.Create(tmpPath)
if err != nil {
return BackupRunResult{}, err
}
defer func() {
_ = tmpFile.Close()
_ = os.Remove(tmpPath)
}()
remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths)
if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err)
}
if _, err := tmpFile.Seek(0, io.SeekStart); err != nil {
return BackupRunResult{}, err
}
st, _ := tmpFile.Stat()
size := int64(0)
if st != nil {
size = st.Size()
}
uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size)
if err != nil {
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "failed"
plan.LastError = err.Error()
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
_ = s.store.Save(reg)
return BackupRunResult{}, fmt.Errorf("upload backup: %w", err)
}
plan.LastRunAt = s.now().UTC()
plan.LastStatus = "ok"
plan.LastError = ""
plan.LastArchive = archiveName
plan.UpdatedAt = s.now().UTC()
reg.Backups = cfg
if err := s.store.Save(reg); err != nil {
return BackupRunResult{}, err
}
_ = s.AppendChange("backup.plan.run", plan.ID, archiveName)
return BackupRunResult{
PlanID: plan.ID,
PlanName: plan.Name,
TargetID: target.ID,
TargetName: target.Name,
ArchiveName: archiveName,
UploadedTo: uploadedTo,
SizeBytes: size,
RanAt: plan.LastRunAt,
}, nil
}
func normalizeBackupPaths(in []string) []string {
out := make([]string, 0, len(in))
seen := map[string]struct{}{}
for _, p := range in {
v := strings.TrimSpace(p)
if v == "" {
continue
}
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
out = append(out, v)
}
return out
}
func sanitizeBackupName(v string) string {
v = strings.ToLower(strings.TrimSpace(v))
if v == "" {
return "backup"
}
var b strings.Builder
for _, r := range v {
switch {
case r >= 'a' && r <= 'z':
b.WriteRune(r)
case r >= '0' && r <= '9':
b.WriteRune(r)
case r == '-' || r == '_' || r == '.':
b.WriteRune(r)
default:
b.WriteByte('-')
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
return "backup"
}
return out
}
func buildRemoteTarStreamCommand(paths []string) string {
items := make([]string, 0, len(paths))
for _, p := range paths {
v := strings.TrimSpace(p)
if v == "" {
continue
}
items = append(items, shellQuote(v))
}
if len(items) == 0 {
items = []string{shellQuote("/")}
}
return "tar -czf - " + strings.Join(items, " ")
}
func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error {
session, err := client.NewSession()
if err != nil {
return err
}
defer session.Close()
stdout, err := session.StdoutPipe()
if err != nil {
return err
}
stderr, err := session.StderrPipe()
if err != nil {
return err
}
if err := session.Start("sh -lc " + shellQuote(script)); err != nil {
return err
}
done := make(chan error, 1)
go func() {
_, cpErr := io.Copy(w, stdout)
if cpErr != nil {
done <- cpErr
return
}
done <- session.Wait()
}()
select {
case <-ctx.Done():
_ = session.Close()
return ctx.Err()
case err := <-done:
if err == nil {
return nil
}
b, _ := io.ReadAll(stderr)
msg := strings.TrimSpace(string(b))
if msg == "" {
return err
}
return fmt.Errorf("%w: %s", err, msg)
}
}
func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
switch strings.ToLower(strings.TrimSpace(target.Type)) {
case "sftp":
return uploadBackupSFTP(ctx, target, archiveName, r)
case "s3":
return uploadBackupS3(ctx, target, archiveName, r, size)
default:
return "", fmt.Errorf("unsupported backup target type %q", target.Type)
}
}
func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) {
_ = ctx
host := strings.TrimSpace(t.SFTPHost)
if host == "" {
return "", fmt.Errorf("sftp_host is required")
}
user := strings.TrimSpace(t.SFTPUser)
if user == "" {
return "", fmt.Errorf("sftp_user is required")
}
addr := netJoinHostPort(host, t.SFTPPort)
auths := make([]ssh.AuthMethod, 0, 2)
if strings.TrimSpace(t.SFTPPassword) != "" {
auths = append(auths, ssh.Password(t.SFTPPassword))
}
if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" {
pemBytes, err := os.ReadFile(kp)
if err != nil {
return "", fmt.Errorf("read sftp key: %w", err)
}
signer, err := ssh.ParsePrivateKey(pemBytes)
if err != nil {
return "", fmt.Errorf("parse sftp key: %w", err)
}
auths = append(auths, ssh.PublicKeys(signer))
}
if len(auths) == 0 {
return "", fmt.Errorf("sftp auth is required (password or key)")
}
sshCfg := &ssh.ClientConfig{
User: user,
Auth: auths,
HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust
Timeout: 15 * time.Second,
}
conn, err := ssh.Dial("tcp", addr, sshCfg)
if err != nil {
return "", err
}
defer conn.Close()
c, err := sftp.NewClient(conn)
if err != nil {
return "", err
}
defer c.Close()
base := strings.TrimSpace(t.SFTPBasePath)
if base == "" {
base = "."
}
if err := c.MkdirAll(base); err != nil {
return "", err
}
remote := path.Join(base, archiveName)
f, err := c.Create(remote)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(f, r); err != nil {
return "", err
}
return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil
}
func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) {
endpoint := strings.TrimSpace(t.S3Endpoint)
bucket := strings.TrimSpace(t.S3Bucket)
access := strings.TrimSpace(t.S3AccessKey)
secret := strings.TrimSpace(t.S3SecretKey)
if endpoint == "" || bucket == "" || access == "" || secret == "" {
return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required")
}
region := strings.TrimSpace(t.S3Region)
if region == "" {
region = "us-east-1"
}
lookup := minio.BucketLookupAuto
if t.S3PathStyle {
lookup = minio.BucketLookupPath
}
cli, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(access, secret, ""),
Secure: t.S3UseSSL,
Region: region,
BucketLookup: lookup,
})
if err != nil {
return "", err
}
key := archiveName
if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" {
key = p + "/" + archiveName
}
opts := minio.PutObjectOptions{ContentType: "application/gzip"}
if size < 0 {
size = -1
}
_, err = cli.PutObject(ctx, bucket, key, r, size, opts)
if err != nil {
return "", err
}
scheme := "https"
if !t.S3UseSSL {
scheme = "http"
}
return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil
}
func netJoinHostPort(host string, port int) string {
p := port
if p <= 0 {
p = 22
}
return net.JoinHostPort(host, strconv.Itoa(p))
}