package cluster import ( "context" "fmt" "io" "net" "os" "path" "path/filepath" "strconv" "strings" "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) type BackupRunResult struct { PlanID string `json:"plan_id"` PlanName string `json:"plan_name"` TargetID string `json:"target_id"` TargetName string `json:"target_name"` ArchiveName string `json:"archive_name"` UploadedTo string `json:"uploaded_to"` SizeBytes int64 `json:"size_bytes"` RanAt time.Time `json:"ran_at"` } func (s *Service) BackupListTargets() ([]BackupTarget, error) { reg, err := s.store.Load() if err != nil { return nil, err } return append([]BackupTarget(nil), normalizeBackupConfig(reg.Backups).Targets...), nil } func (s *Service) BackupListPlans() ([]BackupPlan, error) { reg, err := s.store.Load() if err != nil { return nil, err } return append([]BackupPlan(nil), normalizeBackupConfig(reg.Backups).Plans...), nil } func (s *Service) BackupTestTarget(ctx context.Context, selector string) (string, error) { reg, err := s.store.Load() if err != nil { return "", err } cfg := normalizeBackupConfig(reg.Backups) selector = strings.TrimSpace(selector) var target *BackupTarget for i := range cfg.Targets { if strings.EqualFold(cfg.Targets[i].ID, selector) || strings.EqualFold(cfg.Targets[i].Name, selector) { target = &cfg.Targets[i] break } } if target == nil { return "", fmt.Errorf("backup target not found") } switch target.Type { case "sftp": host := strings.TrimSpace(target.SFTPHost) user := strings.TrimSpace(target.SFTPUser) if host == "" || user == "" { return "", fmt.Errorf("sftp target has empty host/user") } addr := netJoinHostPort(host, target.SFTPPort) auths := make([]ssh.AuthMethod, 0, 2) if strings.TrimSpace(target.SFTPPassword) != "" { auths = append(auths, ssh.Password(target.SFTPPassword)) } if kp := strings.TrimSpace(target.SFTPKeyPath); kp != "" { pemBytes, err := os.ReadFile(kp) if err != nil { return "", fmt.Errorf("read sftp key: %w", err) } signer, err := ssh.ParsePrivateKey(pemBytes) if err != nil { return "", fmt.Errorf("parse sftp key: %w", err) } auths = append(auths, ssh.PublicKeys(signer)) } if len(auths) == 0 { return "", fmt.Errorf("sftp auth is required (password or key)") } sshCfg := &ssh.ClientConfig{ User: user, Auth: auths, HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: 15 * time.Second, } conn, err := ssh.Dial("tcp", addr, sshCfg) if err != nil { return "", err } defer conn.Close() c, err := sftp.NewClient(conn) if err != nil { return "", err } defer c.Close() base := strings.TrimSpace(target.SFTPBasePath) if base == "" { base = "." } if err := c.MkdirAll(base); err != nil { return "", err } if _, err := c.ReadDir(base); err != nil { return "", err } return "sftp://" + addr + "/" + strings.TrimLeft(base, "/"), nil case "s3": endpoint := strings.TrimSpace(target.S3Endpoint) bucket := strings.TrimSpace(target.S3Bucket) access := strings.TrimSpace(target.S3AccessKey) secret := strings.TrimSpace(target.S3SecretKey) if endpoint == "" || bucket == "" || access == "" || secret == "" { return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required") } region := strings.TrimSpace(target.S3Region) if region == "" { region = "us-east-1" } lookup := minio.BucketLookupAuto if target.S3PathStyle { lookup = minio.BucketLookupPath } cli, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(access, secret, ""), Secure: target.S3UseSSL, Region: region, BucketLookup: lookup, }) if err != nil { return "", err } exists, err := cli.BucketExists(ctx, bucket) if err != nil { return "", err } if !exists { return "", fmt.Errorf("bucket %q does not exist or is not accessible", bucket) } scheme := "https" if !target.S3UseSSL { scheme = "http" } return scheme + "://" + endpoint + "/" + bucket, nil default: return "", fmt.Errorf("unsupported target type %q", target.Type) } } func (s *Service) BackupAddTarget(t BackupTarget) (BackupTarget, error) { reg, err := s.store.Load() if err != nil { return BackupTarget{}, err } cfg := normalizeBackupConfig(reg.Backups) t.ID = strings.TrimSpace(t.ID) if t.ID == "" { t.ID = newClusterID() } t.Name = strings.TrimSpace(t.Name) if t.Name == "" { return BackupTarget{}, fmt.Errorf("target name is required") } t.Type = strings.ToLower(strings.TrimSpace(t.Type)) if t.Type != "sftp" && t.Type != "s3" { return BackupTarget{}, fmt.Errorf("target type must be sftp|s3") } if t.Type == "sftp" { if strings.TrimSpace(t.SFTPHost) == "" || strings.TrimSpace(t.SFTPUser) == "" { return BackupTarget{}, fmt.Errorf("sftp target requires --sftp-host and --sftp-user") } if strings.TrimSpace(t.SFTPPassword) == "" && strings.TrimSpace(t.SFTPKeyPath) == "" { return BackupTarget{}, fmt.Errorf("sftp target requires password or key") } } if t.Type == "s3" { if strings.TrimSpace(t.S3Endpoint) == "" || strings.TrimSpace(t.S3Bucket) == "" { return BackupTarget{}, fmt.Errorf("s3 target requires --s3-endpoint and --s3-bucket") } if strings.TrimSpace(t.S3AccessKey) == "" || strings.TrimSpace(t.S3SecretKey) == "" { return BackupTarget{}, fmt.Errorf("s3 target requires --s3-access-key and --s3-secret-key") } } for _, ex := range cfg.Targets { if strings.EqualFold(ex.Name, t.Name) { return BackupTarget{}, fmt.Errorf("target %q already exists", t.Name) } } now := s.now().UTC() t.CreatedAt = now t.UpdatedAt = now if t.SFTPPort <= 0 { t.SFTPPort = 22 } if !t.Enabled { t.Enabled = true } cfg.Targets = append(cfg.Targets, t) reg.Backups = cfg if err := s.store.Save(reg); err != nil { return BackupTarget{}, err } _ = s.AppendChange("backup.target.add", t.ID, t.Name) return t, nil } func (s *Service) BackupRemoveTarget(selector string) (BackupTarget, error) { reg, err := s.store.Load() if err != nil { return BackupTarget{}, err } cfg := normalizeBackupConfig(reg.Backups) selector = strings.TrimSpace(selector) idx := -1 for i, t := range cfg.Targets { if strings.EqualFold(t.ID, selector) || strings.EqualFold(t.Name, selector) { idx = i break } } if idx < 0 { return BackupTarget{}, fmt.Errorf("backup target not found") } removed := cfg.Targets[idx] cfg.Targets = append(cfg.Targets[:idx], cfg.Targets[idx+1:]...) reg.Backups = cfg if err := s.store.Save(reg); err != nil { return BackupTarget{}, err } _ = s.AppendChange("backup.target.remove", removed.ID, removed.Name) return removed, nil } func (s *Service) BackupAddPlan(p BackupPlan) (BackupPlan, error) { reg, err := s.store.Load() if err != nil { return BackupPlan{}, err } cfg := normalizeBackupConfig(reg.Backups) if strings.TrimSpace(p.Name) == "" { return BackupPlan{}, fmt.Errorf("plan name is required") } if strings.TrimSpace(p.TargetID) == "" { return BackupPlan{}, fmt.Errorf("target is required") } if len(p.Paths) == 0 { return BackupPlan{}, fmt.Errorf("at least one path is required") } targetID := "" for _, t := range cfg.Targets { if strings.EqualFold(t.ID, p.TargetID) || strings.EqualFold(t.Name, p.TargetID) { targetID = t.ID break } } if targetID == "" { return BackupPlan{}, fmt.Errorf("backup target %q not found", p.TargetID) } for _, ex := range cfg.Plans { if strings.EqualFold(ex.Name, p.Name) { return BackupPlan{}, fmt.Errorf("plan %q already exists", p.Name) } } p.ID = newClusterID() p.TargetID = targetID p.Paths = normalizeBackupPaths(p.Paths) if strings.TrimSpace(p.Every) == "" { p.Every = "24h" } if p.RetainDays <= 0 { p.RetainDays = 30 } p.Compress = true now := s.now().UTC() p.CreatedAt = now p.UpdatedAt = now if !p.Enabled { p.Enabled = true } cfg.Plans = append(cfg.Plans, p) reg.Backups = cfg if err := s.store.Save(reg); err != nil { return BackupPlan{}, err } _ = s.AppendChange("backup.plan.add", p.ID, p.Name) return p, nil } func (s *Service) BackupRemovePlan(selector string) (BackupPlan, error) { reg, err := s.store.Load() if err != nil { return BackupPlan{}, err } cfg := normalizeBackupConfig(reg.Backups) selector = strings.TrimSpace(selector) idx := -1 for i, p := range cfg.Plans { if strings.EqualFold(p.ID, selector) || strings.EqualFold(p.Name, selector) { idx = i break } } if idx < 0 { return BackupPlan{}, fmt.Errorf("backup plan not found") } removed := cfg.Plans[idx] cfg.Plans = append(cfg.Plans[:idx], cfg.Plans[idx+1:]...) reg.Backups = cfg if err := s.store.Save(reg); err != nil { return BackupPlan{}, err } _ = s.AppendChange("backup.plan.remove", removed.ID, removed.Name) return removed, nil } func (s *Service) BackupRunPlan(ctx context.Context, selector string) (BackupRunResult, error) { reg, err := s.store.Load() if err != nil { return BackupRunResult{}, err } cfg := normalizeBackupConfig(reg.Backups) var plan *BackupPlan for i := range cfg.Plans { if strings.EqualFold(cfg.Plans[i].ID, selector) || strings.EqualFold(cfg.Plans[i].Name, selector) { plan = &cfg.Plans[i] break } } if plan == nil { return BackupRunResult{}, fmt.Errorf("backup plan %q not found", selector) } var target *BackupTarget for i := range cfg.Targets { if cfg.Targets[i].ID == plan.TargetID { target = &cfg.Targets[i] break } } if target == nil { return BackupRunResult{}, fmt.Errorf("backup target %q not found", plan.TargetID) } c, err := s.Get(plan.Cluster) if err != nil { return BackupRunResult{}, err } sshClient, err := s.dialSSH(ctx, c, "", "") if err != nil { return BackupRunResult{}, fmt.Errorf("backup ssh connect: %w", err) } defer sshClient.Close() ts := s.now().UTC().Format("20060102T150405Z") archiveName := sanitizeBackupName(plan.Name) + "-" + sanitizeBackupName(c.Name) + "-" + ts + ".tar.gz" tmpPath := filepath.Join(os.TempDir(), archiveName) tmpFile, err := os.Create(tmpPath) if err != nil { return BackupRunResult{}, err } defer func() { _ = tmpFile.Close() _ = os.Remove(tmpPath) }() remoteTarCmd := buildRemoteTarStreamCommand(plan.Paths) if err := streamRemoteCommandToWriter(ctx, sshClient, remoteTarCmd, tmpFile); err != nil { plan.LastRunAt = s.now().UTC() plan.LastStatus = "failed" plan.LastError = err.Error() plan.UpdatedAt = s.now().UTC() reg.Backups = cfg _ = s.store.Save(reg) return BackupRunResult{}, fmt.Errorf("backup archive stream failed: %w", err) } if _, err := tmpFile.Seek(0, io.SeekStart); err != nil { return BackupRunResult{}, err } st, _ := tmpFile.Stat() size := int64(0) if st != nil { size = st.Size() } uploadedTo, err := uploadBackupObject(ctx, *target, archiveName, tmpFile, size) if err != nil { plan.LastRunAt = s.now().UTC() plan.LastStatus = "failed" plan.LastError = err.Error() plan.UpdatedAt = s.now().UTC() reg.Backups = cfg _ = s.store.Save(reg) return BackupRunResult{}, fmt.Errorf("upload backup: %w", err) } plan.LastRunAt = s.now().UTC() plan.LastStatus = "ok" plan.LastError = "" plan.LastArchive = archiveName plan.UpdatedAt = s.now().UTC() reg.Backups = cfg if err := s.store.Save(reg); err != nil { return BackupRunResult{}, err } _ = s.AppendChange("backup.plan.run", plan.ID, archiveName) return BackupRunResult{ PlanID: plan.ID, PlanName: plan.Name, TargetID: target.ID, TargetName: target.Name, ArchiveName: archiveName, UploadedTo: uploadedTo, SizeBytes: size, RanAt: plan.LastRunAt, }, nil } func normalizeBackupPaths(in []string) []string { out := make([]string, 0, len(in)) seen := map[string]struct{}{} for _, p := range in { v := strings.TrimSpace(p) if v == "" { continue } if _, ok := seen[v]; ok { continue } seen[v] = struct{}{} out = append(out, v) } return out } func sanitizeBackupName(v string) string { v = strings.ToLower(strings.TrimSpace(v)) if v == "" { return "backup" } var b strings.Builder for _, r := range v { switch { case r >= 'a' && r <= 'z': b.WriteRune(r) case r >= '0' && r <= '9': b.WriteRune(r) case r == '-' || r == '_' || r == '.': b.WriteRune(r) default: b.WriteByte('-') } } out := strings.Trim(b.String(), "-") if out == "" { return "backup" } return out } func buildRemoteTarStreamCommand(paths []string) string { items := make([]string, 0, len(paths)) for _, p := range paths { v := strings.TrimSpace(p) if v == "" { continue } items = append(items, shellQuote(v)) } if len(items) == 0 { items = []string{shellQuote("/")} } return "tar -czf - " + strings.Join(items, " ") } func streamRemoteCommandToWriter(ctx context.Context, client *ssh.Client, script string, w io.Writer) error { session, err := client.NewSession() if err != nil { return err } defer session.Close() stdout, err := session.StdoutPipe() if err != nil { return err } stderr, err := session.StderrPipe() if err != nil { return err } if err := session.Start("sh -lc " + shellQuote(script)); err != nil { return err } done := make(chan error, 1) go func() { _, cpErr := io.Copy(w, stdout) if cpErr != nil { done <- cpErr return } done <- session.Wait() }() select { case <-ctx.Done(): _ = session.Close() return ctx.Err() case err := <-done: if err == nil { return nil } b, _ := io.ReadAll(stderr) msg := strings.TrimSpace(string(b)) if msg == "" { return err } return fmt.Errorf("%w: %s", err, msg) } } func uploadBackupObject(ctx context.Context, target BackupTarget, archiveName string, r io.Reader, size int64) (string, error) { switch strings.ToLower(strings.TrimSpace(target.Type)) { case "sftp": return uploadBackupSFTP(ctx, target, archiveName, r) case "s3": return uploadBackupS3(ctx, target, archiveName, r, size) default: return "", fmt.Errorf("unsupported backup target type %q", target.Type) } } func uploadBackupSFTP(ctx context.Context, t BackupTarget, archiveName string, r io.Reader) (string, error) { _ = ctx host := strings.TrimSpace(t.SFTPHost) if host == "" { return "", fmt.Errorf("sftp_host is required") } user := strings.TrimSpace(t.SFTPUser) if user == "" { return "", fmt.Errorf("sftp_user is required") } addr := netJoinHostPort(host, t.SFTPPort) auths := make([]ssh.AuthMethod, 0, 2) if strings.TrimSpace(t.SFTPPassword) != "" { auths = append(auths, ssh.Password(t.SFTPPassword)) } if kp := strings.TrimSpace(t.SFTPKeyPath); kp != "" { pemBytes, err := os.ReadFile(kp) if err != nil { return "", fmt.Errorf("read sftp key: %w", err) } signer, err := ssh.ParsePrivateKey(pemBytes) if err != nil { return "", fmt.Errorf("parse sftp key: %w", err) } auths = append(auths, ssh.PublicKeys(signer)) } if len(auths) == 0 { return "", fmt.Errorf("sftp auth is required (password or key)") } sshCfg := &ssh.ClientConfig{ User: user, Auth: auths, HostKeyCallback: ssh.InsecureIgnoreHostKey(), // external storage endpoint; user-managed trust Timeout: 15 * time.Second, } conn, err := ssh.Dial("tcp", addr, sshCfg) if err != nil { return "", err } defer conn.Close() c, err := sftp.NewClient(conn) if err != nil { return "", err } defer c.Close() base := strings.TrimSpace(t.SFTPBasePath) if base == "" { base = "." } if err := c.MkdirAll(base); err != nil { return "", err } remote := path.Join(base, archiveName) f, err := c.Create(remote) if err != nil { return "", err } defer f.Close() if _, err := io.Copy(f, r); err != nil { return "", err } return "sftp://" + addr + "/" + strings.TrimLeft(remote, "/"), nil } func uploadBackupS3(ctx context.Context, t BackupTarget, archiveName string, r io.Reader, size int64) (string, error) { endpoint := strings.TrimSpace(t.S3Endpoint) bucket := strings.TrimSpace(t.S3Bucket) access := strings.TrimSpace(t.S3AccessKey) secret := strings.TrimSpace(t.S3SecretKey) if endpoint == "" || bucket == "" || access == "" || secret == "" { return "", fmt.Errorf("s3 endpoint/bucket/access/secret are required") } region := strings.TrimSpace(t.S3Region) if region == "" { region = "us-east-1" } lookup := minio.BucketLookupAuto if t.S3PathStyle { lookup = minio.BucketLookupPath } cli, err := minio.New(endpoint, &minio.Options{ Creds: credentials.NewStaticV4(access, secret, ""), Secure: t.S3UseSSL, Region: region, BucketLookup: lookup, }) if err != nil { return "", err } key := archiveName if p := strings.Trim(strings.TrimSpace(t.S3Prefix), "/"); p != "" { key = p + "/" + archiveName } opts := minio.PutObjectOptions{ContentType: "application/gzip"} if size < 0 { size = -1 } _, err = cli.PutObject(ctx, bucket, key, r, size, opts) if err != nil { return "", err } scheme := "https" if !t.S3UseSSL { scheme = "http" } return scheme + "://" + endpoint + "/" + bucket + "/" + key, nil } func netJoinHostPort(host string, port int) string { p := port if p <= 0 { p = 22 } return net.JoinHostPort(host, strconv.Itoa(p)) }