138 lines
3.8 KiB
Go
138 lines
3.8 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type VMStateSummary struct {
|
|
Total int `json:"total"`
|
|
Running int `json:"running"`
|
|
ShutOff int `json:"shut_off"`
|
|
Paused int `json:"paused"`
|
|
Others int `json:"others"`
|
|
ShutOffNames []string `json:"shut_off_names,omitempty"`
|
|
PausedNames []string `json:"paused_names,omitempty"`
|
|
OtherNames []string `json:"other_names,omitempty"`
|
|
Warnings []string `json:"warnings,omitempty"`
|
|
SampledAt time.Time `json:"sampled_at"`
|
|
}
|
|
|
|
func (s *Service) GetVMAlertPolicy(selector string) (VMAlertPolicy, error) {
|
|
c, err := s.Get(selector)
|
|
if err != nil {
|
|
return VMAlertPolicy{}, err
|
|
}
|
|
return ensureVMAlertPolicy(c.VMAlerts), nil
|
|
}
|
|
|
|
func (s *Service) SetVMAlertPolicy(selector string, p VMAlertPolicy) (Cluster, error) {
|
|
reg, err := s.store.Load()
|
|
if err != nil {
|
|
return Cluster{}, err
|
|
}
|
|
c, idx, err := findCluster(reg, selector)
|
|
if err != nil {
|
|
return Cluster{}, err
|
|
}
|
|
c.VMAlerts = ensureVMAlertPolicy(p)
|
|
c.UpdatedAt = s.now().UTC()
|
|
reg.Clusters[idx] = c
|
|
if err := s.store.Save(reg); err != nil {
|
|
return Cluster{}, err
|
|
}
|
|
_ = s.AppendChange("vm.alerts", c.Name, fmt.Sprintf("enabled=%t warn_on_shutoff=%t min_running=%d", c.VMAlerts.Enabled, c.VMAlerts.WarnOnShutoff, c.VMAlerts.MinRunning))
|
|
return c, nil
|
|
}
|
|
|
|
func (s *Service) CheckVMAlerts(ctx context.Context, selector string) (VMStateSummary, error) {
|
|
c, err := s.Get(selector)
|
|
if err != nil {
|
|
return VMStateSummary{}, err
|
|
}
|
|
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
|
|
if err != nil {
|
|
return VMStateSummary{}, err
|
|
}
|
|
states := parseVirshListStates(out)
|
|
summary := VMStateSummary{SampledAt: s.now().UTC(), Total: len(states)}
|
|
for name, st := range states {
|
|
n := strings.ToLower(strings.TrimSpace(st))
|
|
switch {
|
|
case n == "running":
|
|
summary.Running++
|
|
case n == "shut off":
|
|
summary.ShutOff++
|
|
summary.ShutOffNames = append(summary.ShutOffNames, name)
|
|
case n == "paused":
|
|
summary.Paused++
|
|
summary.PausedNames = append(summary.PausedNames, name)
|
|
default:
|
|
summary.Others++
|
|
summary.OtherNames = append(summary.OtherNames, name)
|
|
}
|
|
}
|
|
sort.Strings(summary.ShutOffNames)
|
|
sort.Strings(summary.PausedNames)
|
|
sort.Strings(summary.OtherNames)
|
|
p := ensureVMAlertPolicy(c.VMAlerts)
|
|
if p.Enabled {
|
|
if p.WarnOnShutoff && summary.ShutOff > 0 {
|
|
summary.Warnings = append(summary.Warnings, fmt.Sprintf(
|
|
"%d VM(s) are shut off: %s",
|
|
summary.ShutOff,
|
|
joinNamesLimit(summary.ShutOffNames, 12),
|
|
))
|
|
}
|
|
if summary.Running < p.MinRunning {
|
|
summary.Warnings = append(summary.Warnings, fmt.Sprintf("running VM count %d is below min_running=%d", summary.Running, p.MinRunning))
|
|
}
|
|
}
|
|
sort.Strings(summary.Warnings)
|
|
return summary, nil
|
|
}
|
|
|
|
func (s *Service) ListVMStates(ctx context.Context, selector string) (map[string]string, error) {
|
|
c, err := s.Get(selector)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out, err := s.RunPluginAction(ctx, c.ID, "kvm", "list", nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return parseVirshListStates(out), nil
|
|
}
|
|
|
|
func parseVirshListStates(raw string) map[string]string {
|
|
lines := strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n")
|
|
out := make(map[string]string)
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "Id") || strings.HasPrefix(line, "-") {
|
|
continue
|
|
}
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 3 {
|
|
continue
|
|
}
|
|
name := fields[1]
|
|
state := strings.Join(fields[2:], " ")
|
|
out[name] = state
|
|
}
|
|
return out
|
|
}
|
|
|
|
func joinNamesLimit(items []string, limit int) string {
|
|
if len(items) == 0 {
|
|
return "-"
|
|
}
|
|
if limit <= 0 || len(items) <= limit {
|
|
return strings.Join(items, ", ")
|
|
}
|
|
return strings.Join(items[:limit], ", ") + fmt.Sprintf(" (+%d more)", len(items)-limit)
|
|
}
|