498 lines
12 KiB
Go
498 lines
12 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
"github.com/charmbracelet/lipgloss"
|
|
|
|
"pxmon/internal/cluster"
|
|
"pxmon/internal/history"
|
|
)
|
|
|
|
// liveClusterStat is one row on the live dashboard. Fields are best-effort;
|
|
// if the agent is unreachable we still render the row with an error so the
|
|
// operator sees the node exists but is degraded.
|
|
type liveClusterStat struct {
|
|
ClusterID string
|
|
Name string
|
|
Host string
|
|
Reachable bool
|
|
Err string
|
|
CPUPercent float64
|
|
MemPercent float64
|
|
MemUsed uint64
|
|
MemTotal uint64
|
|
DiskUsePct float64
|
|
NetMbps float64 // current: last point from history
|
|
P95Mbps float64 // primary iface, 24h window
|
|
Uplink string
|
|
HostName string
|
|
}
|
|
|
|
type monitorLiveMsg struct {
|
|
entries []liveClusterStat
|
|
err error
|
|
}
|
|
|
|
type liveAutoTickMsg struct{}
|
|
|
|
func liveAutoTickCmd() tea.Cmd {
|
|
return tea.Tick(liveAutoTickInterval, func(_ time.Time) tea.Msg {
|
|
return liveAutoTickMsg{}
|
|
})
|
|
}
|
|
|
|
const (
|
|
livePageSize = 14
|
|
liveFetchWorkers = 16
|
|
liveAutoTickInterval = 5 * time.Second
|
|
liveP95HistoryWindow = 24 * time.Hour
|
|
liveFetchClusterTO = 5 * time.Second
|
|
liveFetchOverallTO = 25 * time.Second
|
|
)
|
|
|
|
// fetchLiveCmd gathers live stats for every cluster with an installed agent
|
|
// in parallel, pairs each with P95 from history, and emits a sorted slice.
|
|
func (m monitorModel) fetchLiveCmd() tea.Cmd {
|
|
svc := m.svc
|
|
return func() tea.Msg {
|
|
clusters, _, err := svc.List()
|
|
if err != nil {
|
|
return monitorLiveMsg{err: err}
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), liveFetchOverallTO)
|
|
defer cancel()
|
|
|
|
type work struct {
|
|
idx int
|
|
c cluster.Cluster
|
|
}
|
|
ch := make(chan work, len(clusters))
|
|
results := make([]liveClusterStat, len(clusters))
|
|
|
|
var wg sync.WaitGroup
|
|
workers := liveFetchWorkers
|
|
if workers > len(clusters) {
|
|
workers = len(clusters)
|
|
}
|
|
if workers < 1 {
|
|
workers = 1
|
|
}
|
|
|
|
for w := 0; w < workers; w++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for item := range ch {
|
|
results[item.idx] = collectLiveRow(ctx, svc, item.c)
|
|
}
|
|
}()
|
|
}
|
|
for i, c := range clusters {
|
|
ch <- work{idx: i, c: c}
|
|
}
|
|
close(ch)
|
|
wg.Wait()
|
|
|
|
out := make([]liveClusterStat, 0, len(results))
|
|
for _, r := range results {
|
|
if r.ClusterID == "" {
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return monitorLiveMsg{entries: out}
|
|
}
|
|
}
|
|
|
|
func collectLiveRow(ctx context.Context, svc *cluster.Service, c cluster.Cluster) liveClusterStat {
|
|
row := liveClusterStat{
|
|
ClusterID: c.ID,
|
|
Name: c.Name,
|
|
Host: c.Host,
|
|
}
|
|
if !c.Agent.Installed {
|
|
row.Err = "agent not installed"
|
|
return row
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, liveFetchClusterTO)
|
|
defer cancel()
|
|
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
|
|
if err != nil {
|
|
row.Err = err.Error()
|
|
return row
|
|
}
|
|
row.Reachable = true
|
|
row.CPUPercent = stats.CPU.UsagePercent
|
|
row.MemPercent = stats.Memory.UsedPercent
|
|
row.MemUsed = stats.Memory.UsedBytes
|
|
row.MemTotal = stats.Memory.TotalBytes
|
|
row.HostName = stats.Host.Hostname
|
|
var maxDisk float64
|
|
for _, d := range stats.Disk {
|
|
if d.UsedPercent > maxDisk {
|
|
maxDisk = d.UsedPercent
|
|
}
|
|
}
|
|
row.DiskUsePct = maxDisk
|
|
|
|
// Historical P95 from the persisted store (populated by the sampler
|
|
// running inside TUI or bot daemon).
|
|
if store := svc.NetworkStore(); store != nil {
|
|
since := time.Now().Add(-liveP95HistoryWindow)
|
|
if snaps, err := store.Load(c.ID, since); err == nil && len(snaps) > 0 {
|
|
primary := history.PrimaryInterface(snaps)
|
|
row.Uplink = primary
|
|
series := history.AggregateNodeSeries(snaps, primary)
|
|
if len(series) > 0 {
|
|
row.P95Mbps = history.PercentileMbps(series, 95)
|
|
row.NetMbps = series[len(series)-1].TotalMbps
|
|
}
|
|
}
|
|
}
|
|
|
|
// Fallback / live reading: take a second stats snapshot after a short
|
|
// pause and derive the instantaneous per-interface Mbps ourselves.
|
|
// This makes NET Mbps meaningful even when the history store is empty
|
|
// (e.g. the sampler hasn't been running long).
|
|
time.Sleep(700 * time.Millisecond)
|
|
callCtx2, cancel2 := context.WithTimeout(ctx, liveFetchClusterTO)
|
|
defer cancel2()
|
|
stats2, err := svc.AgentStatsTyped(callCtx2, c.ID)
|
|
if err != nil {
|
|
return row
|
|
}
|
|
elapsed := stats2.Timestamp.Sub(stats.Timestamp).Seconds()
|
|
if elapsed <= 0 {
|
|
elapsed = 0.7
|
|
}
|
|
prev := make(map[string]struct {
|
|
rx, tx uint64
|
|
}, len(stats.Network))
|
|
for _, n := range stats.Network {
|
|
prev[n.Interface] = struct{ rx, tx uint64 }{n.RxBytes, n.TxBytes}
|
|
}
|
|
bestName := ""
|
|
bestMbps := -1.0
|
|
for _, n := range stats2.Network {
|
|
if isLiveVirtual(n.Interface) {
|
|
continue
|
|
}
|
|
p, ok := prev[n.Interface]
|
|
if !ok {
|
|
continue
|
|
}
|
|
var rxB, txB uint64
|
|
if n.RxBytes >= p.rx {
|
|
rxB = n.RxBytes - p.rx
|
|
}
|
|
if n.TxBytes >= p.tx {
|
|
txB = n.TxBytes - p.tx
|
|
}
|
|
rxMbps := float64(rxB) * 8 / elapsed / 1_000_000
|
|
txMbps := float64(txB) * 8 / elapsed / 1_000_000
|
|
m := rxMbps
|
|
if txMbps > m {
|
|
m = txMbps
|
|
}
|
|
if m > bestMbps {
|
|
bestMbps = m
|
|
bestName = n.Interface
|
|
}
|
|
}
|
|
if bestMbps >= 0 {
|
|
row.NetMbps = bestMbps
|
|
if row.Uplink == "" {
|
|
row.Uplink = bestName
|
|
}
|
|
}
|
|
return row
|
|
}
|
|
|
|
// isLiveVirtual mirrors history.isVirtualIface but is local so we don't
|
|
// export the original.
|
|
func isLiveVirtual(name string) bool {
|
|
n := strings.ToLower(name)
|
|
if n == "" || n == "lo" {
|
|
return true
|
|
}
|
|
prefixes := []string{
|
|
"lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr",
|
|
"cni", "flannel", "wg", "tun", "tailscale", "zt", "kube",
|
|
"cilium", "ovs", "podman", "dummy",
|
|
}
|
|
for _, p := range prefixes {
|
|
if strings.HasPrefix(n, p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// handleLiveKey routes keys while the live dashboard is visible.
|
|
func (m monitorModel) handleLiveKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
|
|
switch v.String() {
|
|
case "o", "esc":
|
|
m.view = viewOverview
|
|
return m, nil
|
|
case "c":
|
|
m.view = viewClusters
|
|
return m, nil
|
|
case "r", "R":
|
|
m.liveLoading = true
|
|
m.setStatus("live: refreshing")
|
|
return m, m.fetchLiveCmd()
|
|
case "up", "k":
|
|
if m.liveCursor > 0 {
|
|
m.liveCursor--
|
|
}
|
|
return m, nil
|
|
case "down", "j":
|
|
if m.liveCursor < len(m.liveEntries)-1 {
|
|
m.liveCursor++
|
|
}
|
|
return m, nil
|
|
case "pgup":
|
|
if m.livePage > 0 {
|
|
m.livePage--
|
|
}
|
|
return m, nil
|
|
case "pgdown":
|
|
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
|
|
if m.livePage < pages-1 {
|
|
m.livePage++
|
|
}
|
|
return m, nil
|
|
case "home":
|
|
m.liveCursor = 0
|
|
m.livePage = 0
|
|
return m, nil
|
|
case "end":
|
|
m.liveCursor = len(m.liveEntries) - 1
|
|
if m.liveCursor < 0 {
|
|
m.liveCursor = 0
|
|
}
|
|
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
|
|
if pages > 0 {
|
|
m.livePage = pages - 1
|
|
}
|
|
return m, nil
|
|
case "enter":
|
|
if m.livePinned == nil {
|
|
m.livePinned = map[string]bool{}
|
|
}
|
|
rows := m.sortedLiveEntries()
|
|
if m.liveCursor >= 0 && m.liveCursor < len(rows) {
|
|
id := rows[m.liveCursor].ClusterID
|
|
if m.livePinned[id] {
|
|
delete(m.livePinned, id)
|
|
m.setStatus("live: unpinned " + rows[m.liveCursor].Name)
|
|
} else {
|
|
m.livePinned[id] = true
|
|
m.setStatus("live: pinned " + rows[m.liveCursor].Name)
|
|
}
|
|
}
|
|
return m, nil
|
|
case "1":
|
|
m.liveSort = 0
|
|
return m, nil
|
|
case "2":
|
|
m.liveSort = 1
|
|
return m, nil
|
|
case "3":
|
|
m.liveSort = 2
|
|
return m, nil
|
|
case "4":
|
|
m.liveSort = 3
|
|
return m, nil
|
|
case "5":
|
|
m.liveSort = 4
|
|
return m, nil
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
// sortedLiveEntries returns entries sorted by the active metric. Pinned
|
|
// clusters always float to the top so the operator keeps eyes on them even
|
|
// when their usage changes.
|
|
func (m monitorModel) sortedLiveEntries() []liveClusterStat {
|
|
entries := make([]liveClusterStat, len(m.liveEntries))
|
|
copy(entries, m.liveEntries)
|
|
key := func(r liveClusterStat) float64 {
|
|
switch m.liveSort {
|
|
case 1:
|
|
return r.MemPercent
|
|
case 2:
|
|
return r.NetMbps
|
|
case 3:
|
|
return r.P95Mbps
|
|
case 4:
|
|
return r.DiskUsePct
|
|
default:
|
|
return r.CPUPercent
|
|
}
|
|
}
|
|
sort.SliceStable(entries, func(i, j int) bool {
|
|
pi := m.livePinned[entries[i].ClusterID]
|
|
pj := m.livePinned[entries[j].ClusterID]
|
|
if pi != pj {
|
|
return pi
|
|
}
|
|
return key(entries[i]) > key(entries[j])
|
|
})
|
|
return entries
|
|
}
|
|
|
|
func liveSortLabel(n int) string {
|
|
switch n {
|
|
case 1:
|
|
return "ram"
|
|
case 2:
|
|
return "net"
|
|
case 3:
|
|
return "p95"
|
|
case 4:
|
|
return "disk"
|
|
default:
|
|
return "cpu"
|
|
}
|
|
}
|
|
|
|
// renderLiveDashboard paints the paginated live view.
|
|
func (m monitorModel) renderLiveDashboard(width int) string {
|
|
title := titleStyle.Render("━━ Live cluster fleet ━━")
|
|
help := dimStyle.Render("sort: 1=cpu 2=ram 3=net 4=p95 5=disk enter pin r refresh ↑/↓ select pgup/pgdn page o back")
|
|
|
|
if m.liveLoading && len(m.liveEntries) == 0 {
|
|
body := dimStyle.Render("collecting stats from every cluster…")
|
|
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
|
|
}
|
|
if m.liveError != "" && len(m.liveEntries) == 0 {
|
|
body := critStyle.Render("error: " + m.liveError)
|
|
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
|
|
}
|
|
if len(m.liveEntries) == 0 {
|
|
return lipgloss.JoinVertical(lipgloss.Left, title, help, "",
|
|
dimStyle.Render("no clusters with installed agents"))
|
|
}
|
|
|
|
entries := m.sortedLiveEntries()
|
|
total := len(entries)
|
|
pages := (total + livePageSize - 1) / livePageSize
|
|
if pages == 0 {
|
|
pages = 1
|
|
}
|
|
if m.livePage >= pages {
|
|
m.livePage = pages - 1
|
|
}
|
|
start := m.livePage * livePageSize
|
|
end := start + livePageSize
|
|
if end > total {
|
|
end = total
|
|
}
|
|
page := entries[start:end]
|
|
|
|
headerLine := fmt.Sprintf(" %-3s %-16s %-3s %6s %6s %14s %12s %12s %6s %-10s",
|
|
"#", "NAME", "PIN", "CPU%", "RAM%", "RAM", "NET", "P95", "DISK%", "UPLINK")
|
|
lines := []string{title, help, "",
|
|
dimStyle.Render(fmt.Sprintf("sorted by %s · page %d/%d · %d clusters · refresh every %s · enter=pin",
|
|
liveSortLabel(m.liveSort), m.livePage+1, pages, total, liveAutoTickInterval)),
|
|
"",
|
|
accentStyle.Bold(true).Render(headerLine),
|
|
}
|
|
|
|
for i, row := range page {
|
|
absoluteIdx := start + i
|
|
selected := absoluteIdx == m.liveCursor
|
|
pin := " "
|
|
if m.livePinned[row.ClusterID] {
|
|
pin = "⚑"
|
|
}
|
|
name := row.Name
|
|
if len(name) > 16 {
|
|
name = name[:13] + "..."
|
|
}
|
|
if !row.Reachable {
|
|
line := fmt.Sprintf(" %-3d %-16s %-3s %s",
|
|
absoluteIdx+1, name, pin, critStyle.Render("unreachable: "+clipRight(row.Err, 60)))
|
|
if selected {
|
|
line = selectedRowStyle.Render(line)
|
|
}
|
|
lines = append(lines, line)
|
|
continue
|
|
}
|
|
|
|
cpuStr := colorByThreshold(fmt.Sprintf("%5.1f", row.CPUPercent), row.CPUPercent, 50, 80)
|
|
ramStr := colorByThreshold(fmt.Sprintf("%5.1f", row.MemPercent), row.MemPercent, 50, 80)
|
|
ramAbs := fmt.Sprintf("%7s/%-6s", humanBytesUint(row.MemUsed), humanBytesUint(row.MemTotal))
|
|
netStr := fmt.Sprintf("%12s", formatMbpsHuman(row.NetMbps))
|
|
var p95Str string
|
|
if row.P95Mbps > 0 {
|
|
p95Str = fmt.Sprintf("%12s", formatMbpsHuman(row.P95Mbps))
|
|
} else {
|
|
p95Str = dimStyle.Render(fmt.Sprintf("%12s", "— (no hist)"))
|
|
}
|
|
diskStr := colorByThreshold(fmt.Sprintf("%5.1f", row.DiskUsePct), row.DiskUsePct, 70, 90)
|
|
uplink := row.Uplink
|
|
if uplink == "" {
|
|
uplink = "-"
|
|
}
|
|
if len(uplink) > 10 {
|
|
uplink = uplink[:10]
|
|
}
|
|
|
|
body := fmt.Sprintf(" %-3d %-16s %-3s %s %s %14s %s %s %s %-10s",
|
|
absoluteIdx+1, name, pin,
|
|
cpuStr, ramStr, ramAbs, netStr, p95Str, diskStr, uplink)
|
|
if selected {
|
|
body = selectedRowStyle.Render(body)
|
|
}
|
|
lines = append(lines, body)
|
|
}
|
|
|
|
// Stats footer.
|
|
var totalCPU, totalMem, totalNet, totalP95 float64
|
|
reachable := 0
|
|
for _, r := range entries {
|
|
if !r.Reachable {
|
|
continue
|
|
}
|
|
reachable++
|
|
totalCPU += r.CPUPercent
|
|
totalMem += r.MemPercent
|
|
totalNet += r.NetMbps
|
|
totalP95 += r.P95Mbps
|
|
}
|
|
if reachable > 0 {
|
|
lines = append(lines, "",
|
|
dimStyle.Render(fmt.Sprintf("fleet avg: cpu %.1f%% · ram %.1f%% · net %s · p95 %s · reachable %d/%d",
|
|
totalCPU/float64(reachable),
|
|
totalMem/float64(reachable),
|
|
formatMbpsHuman(totalNet),
|
|
formatMbpsHuman(totalP95),
|
|
reachable, total)))
|
|
}
|
|
|
|
return strings.Join(lines, "\n")
|
|
}
|
|
|
|
var selectedRowStyle = lipgloss.NewStyle().Background(lipgloss.Color("#3A3A3A")).Bold(true)
|
|
|
|
func colorByThreshold(text string, value, warn, crit float64) string {
|
|
switch {
|
|
case value >= crit:
|
|
return critStyle.Render(text)
|
|
case value >= warn:
|
|
return warnStyle.Render(text)
|
|
default:
|
|
return okStyle.Render(text)
|
|
}
|
|
}
|