package cli import ( "context" "fmt" "sort" "strings" "sync" "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "pxmon/internal/cluster" "pxmon/internal/history" ) // liveClusterStat is one row on the live dashboard. Fields are best-effort; // if the agent is unreachable we still render the row with an error so the // operator sees the node exists but is degraded. type liveClusterStat struct { ClusterID string Name string Host string Reachable bool Err string CPUPercent float64 MemPercent float64 MemUsed uint64 MemTotal uint64 DiskUsePct float64 NetMbps float64 // current: last point from history P95Mbps float64 // primary iface, 24h window Uplink string HostName string } type monitorLiveMsg struct { entries []liveClusterStat err error } type liveAutoTickMsg struct{} func liveAutoTickCmd() tea.Cmd { return tea.Tick(liveAutoTickInterval, func(_ time.Time) tea.Msg { return liveAutoTickMsg{} }) } const ( livePageSize = 14 liveFetchWorkers = 16 liveAutoTickInterval = 5 * time.Second liveP95HistoryWindow = 24 * time.Hour liveFetchClusterTO = 5 * time.Second liveFetchOverallTO = 25 * time.Second ) // fetchLiveCmd gathers live stats for every cluster with an installed agent // in parallel, pairs each with P95 from history, and emits a sorted slice. func (m monitorModel) fetchLiveCmd() tea.Cmd { svc := m.svc return func() tea.Msg { clusters, _, err := svc.List() if err != nil { return monitorLiveMsg{err: err} } ctx, cancel := context.WithTimeout(context.Background(), liveFetchOverallTO) defer cancel() type work struct { idx int c cluster.Cluster } ch := make(chan work, len(clusters)) results := make([]liveClusterStat, len(clusters)) var wg sync.WaitGroup workers := liveFetchWorkers if workers > len(clusters) { workers = len(clusters) } if workers < 1 { workers = 1 } for w := 0; w < workers; w++ { wg.Add(1) go func() { defer wg.Done() for item := range ch { results[item.idx] = collectLiveRow(ctx, svc, item.c) } }() } for i, c := range clusters { ch <- work{idx: i, c: c} } close(ch) wg.Wait() out := make([]liveClusterStat, 0, len(results)) for _, r := range results { if r.ClusterID == "" { continue } out = append(out, r) } return monitorLiveMsg{entries: out} } } func collectLiveRow(ctx context.Context, svc *cluster.Service, c cluster.Cluster) liveClusterStat { row := liveClusterStat{ ClusterID: c.ID, Name: c.Name, Host: c.Host, } if !c.Agent.Installed { row.Err = "agent not installed" return row } callCtx, cancel := context.WithTimeout(ctx, liveFetchClusterTO) defer cancel() stats, err := svc.AgentStatsTyped(callCtx, c.ID) if err != nil { row.Err = err.Error() return row } row.Reachable = true row.CPUPercent = stats.CPU.UsagePercent row.MemPercent = stats.Memory.UsedPercent row.MemUsed = stats.Memory.UsedBytes row.MemTotal = stats.Memory.TotalBytes row.HostName = stats.Host.Hostname var maxDisk float64 for _, d := range stats.Disk { if d.UsedPercent > maxDisk { maxDisk = d.UsedPercent } } row.DiskUsePct = maxDisk // Historical P95 from the persisted store (populated by the sampler // running inside TUI or bot daemon). if store := svc.NetworkStore(); store != nil { since := time.Now().Add(-liveP95HistoryWindow) if snaps, err := store.Load(c.ID, since); err == nil && len(snaps) > 0 { primary := history.PrimaryInterface(snaps) row.Uplink = primary series := history.AggregateNodeSeries(snaps, primary) if len(series) > 0 { row.P95Mbps = history.PercentileMbps(series, 95) row.NetMbps = series[len(series)-1].TotalMbps } } } // Fallback / live reading: take a second stats snapshot after a short // pause and derive the instantaneous per-interface Mbps ourselves. // This makes NET Mbps meaningful even when the history store is empty // (e.g. the sampler hasn't been running long). time.Sleep(700 * time.Millisecond) callCtx2, cancel2 := context.WithTimeout(ctx, liveFetchClusterTO) defer cancel2() stats2, err := svc.AgentStatsTyped(callCtx2, c.ID) if err != nil { return row } elapsed := stats2.Timestamp.Sub(stats.Timestamp).Seconds() if elapsed <= 0 { elapsed = 0.7 } prev := make(map[string]struct { rx, tx uint64 }, len(stats.Network)) for _, n := range stats.Network { prev[n.Interface] = struct{ rx, tx uint64 }{n.RxBytes, n.TxBytes} } bestName := "" bestMbps := -1.0 for _, n := range stats2.Network { if isLiveVirtual(n.Interface) { continue } p, ok := prev[n.Interface] if !ok { continue } var rxB, txB uint64 if n.RxBytes >= p.rx { rxB = n.RxBytes - p.rx } if n.TxBytes >= p.tx { txB = n.TxBytes - p.tx } rxMbps := float64(rxB) * 8 / elapsed / 1_000_000 txMbps := float64(txB) * 8 / elapsed / 1_000_000 m := rxMbps if txMbps > m { m = txMbps } if m > bestMbps { bestMbps = m bestName = n.Interface } } if bestMbps >= 0 { row.NetMbps = bestMbps if row.Uplink == "" { row.Uplink = bestName } } return row } // isLiveVirtual mirrors history.isVirtualIface but is local so we don't // export the original. func isLiveVirtual(name string) bool { n := strings.ToLower(name) if n == "" || n == "lo" { return true } prefixes := []string{ "lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr", "cni", "flannel", "wg", "tun", "tailscale", "zt", "kube", "cilium", "ovs", "podman", "dummy", } for _, p := range prefixes { if strings.HasPrefix(n, p) { return true } } return false } // handleLiveKey routes keys while the live dashboard is visible. func (m monitorModel) handleLiveKey(v tea.KeyMsg) (tea.Model, tea.Cmd) { switch v.String() { case "o", "esc": m.view = viewOverview return m, nil case "c": m.view = viewClusters return m, nil case "r", "R": m.liveLoading = true m.setStatus("live: refreshing") return m, m.fetchLiveCmd() case "up", "k": if m.liveCursor > 0 { m.liveCursor-- } return m, nil case "down", "j": if m.liveCursor < len(m.liveEntries)-1 { m.liveCursor++ } return m, nil case "pgup": if m.livePage > 0 { m.livePage-- } return m, nil case "pgdown": pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize if m.livePage < pages-1 { m.livePage++ } return m, nil case "home": m.liveCursor = 0 m.livePage = 0 return m, nil case "end": m.liveCursor = len(m.liveEntries) - 1 if m.liveCursor < 0 { m.liveCursor = 0 } pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize if pages > 0 { m.livePage = pages - 1 } return m, nil case "enter": if m.livePinned == nil { m.livePinned = map[string]bool{} } rows := m.sortedLiveEntries() if m.liveCursor >= 0 && m.liveCursor < len(rows) { id := rows[m.liveCursor].ClusterID if m.livePinned[id] { delete(m.livePinned, id) m.setStatus("live: unpinned " + rows[m.liveCursor].Name) } else { m.livePinned[id] = true m.setStatus("live: pinned " + rows[m.liveCursor].Name) } } return m, nil case "1": m.liveSort = 0 return m, nil case "2": m.liveSort = 1 return m, nil case "3": m.liveSort = 2 return m, nil case "4": m.liveSort = 3 return m, nil case "5": m.liveSort = 4 return m, nil } return m, nil } // sortedLiveEntries returns entries sorted by the active metric. Pinned // clusters always float to the top so the operator keeps eyes on them even // when their usage changes. func (m monitorModel) sortedLiveEntries() []liveClusterStat { entries := make([]liveClusterStat, len(m.liveEntries)) copy(entries, m.liveEntries) key := func(r liveClusterStat) float64 { switch m.liveSort { case 1: return r.MemPercent case 2: return r.NetMbps case 3: return r.P95Mbps case 4: return r.DiskUsePct default: return r.CPUPercent } } sort.SliceStable(entries, func(i, j int) bool { pi := m.livePinned[entries[i].ClusterID] pj := m.livePinned[entries[j].ClusterID] if pi != pj { return pi } return key(entries[i]) > key(entries[j]) }) return entries } func liveSortLabel(n int) string { switch n { case 1: return "ram" case 2: return "net" case 3: return "p95" case 4: return "disk" default: return "cpu" } } // renderLiveDashboard paints the paginated live view. func (m monitorModel) renderLiveDashboard(width int) string { title := titleStyle.Render("━━ Live cluster fleet ━━") help := dimStyle.Render("sort: 1=cpu 2=ram 3=net 4=p95 5=disk enter pin r refresh ↑/↓ select pgup/pgdn page o back") if m.liveLoading && len(m.liveEntries) == 0 { body := dimStyle.Render("collecting stats from every cluster…") return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) } if m.liveError != "" && len(m.liveEntries) == 0 { body := critStyle.Render("error: " + m.liveError) return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body) } if len(m.liveEntries) == 0 { return lipgloss.JoinVertical(lipgloss.Left, title, help, "", dimStyle.Render("no clusters with installed agents")) } entries := m.sortedLiveEntries() total := len(entries) pages := (total + livePageSize - 1) / livePageSize if pages == 0 { pages = 1 } if m.livePage >= pages { m.livePage = pages - 1 } start := m.livePage * livePageSize end := start + livePageSize if end > total { end = total } page := entries[start:end] headerLine := fmt.Sprintf(" %-3s %-16s %-3s %6s %6s %14s %12s %12s %6s %-10s", "#", "NAME", "PIN", "CPU%", "RAM%", "RAM", "NET", "P95", "DISK%", "UPLINK") lines := []string{title, help, "", dimStyle.Render(fmt.Sprintf("sorted by %s · page %d/%d · %d clusters · refresh every %s · enter=pin", liveSortLabel(m.liveSort), m.livePage+1, pages, total, liveAutoTickInterval)), "", accentStyle.Bold(true).Render(headerLine), } for i, row := range page { absoluteIdx := start + i selected := absoluteIdx == m.liveCursor pin := " " if m.livePinned[row.ClusterID] { pin = "⚑" } name := row.Name if len(name) > 16 { name = name[:13] + "..." } if !row.Reachable { line := fmt.Sprintf(" %-3d %-16s %-3s %s", absoluteIdx+1, name, pin, critStyle.Render("unreachable: "+clipRight(row.Err, 60))) if selected { line = selectedRowStyle.Render(line) } lines = append(lines, line) continue } cpuStr := colorByThreshold(fmt.Sprintf("%5.1f", row.CPUPercent), row.CPUPercent, 50, 80) ramStr := colorByThreshold(fmt.Sprintf("%5.1f", row.MemPercent), row.MemPercent, 50, 80) ramAbs := fmt.Sprintf("%7s/%-6s", humanBytesUint(row.MemUsed), humanBytesUint(row.MemTotal)) netStr := fmt.Sprintf("%12s", formatMbpsHuman(row.NetMbps)) var p95Str string if row.P95Mbps > 0 { p95Str = fmt.Sprintf("%12s", formatMbpsHuman(row.P95Mbps)) } else { p95Str = dimStyle.Render(fmt.Sprintf("%12s", "— (no hist)")) } diskStr := colorByThreshold(fmt.Sprintf("%5.1f", row.DiskUsePct), row.DiskUsePct, 70, 90) uplink := row.Uplink if uplink == "" { uplink = "-" } if len(uplink) > 10 { uplink = uplink[:10] } body := fmt.Sprintf(" %-3d %-16s %-3s %s %s %14s %s %s %s %-10s", absoluteIdx+1, name, pin, cpuStr, ramStr, ramAbs, netStr, p95Str, diskStr, uplink) if selected { body = selectedRowStyle.Render(body) } lines = append(lines, body) } // Stats footer. var totalCPU, totalMem, totalNet, totalP95 float64 reachable := 0 for _, r := range entries { if !r.Reachable { continue } reachable++ totalCPU += r.CPUPercent totalMem += r.MemPercent totalNet += r.NetMbps totalP95 += r.P95Mbps } if reachable > 0 { lines = append(lines, "", dimStyle.Render(fmt.Sprintf("fleet avg: cpu %.1f%% · ram %.1f%% · net %s · p95 %s · reachable %d/%d", totalCPU/float64(reachable), totalMem/float64(reachable), formatMbpsHuman(totalNet), formatMbpsHuman(totalP95), reachable, total))) } return strings.Join(lines, "\n") } var selectedRowStyle = lipgloss.NewStyle().Background(lipgloss.Color("#3A3A3A")).Bold(true) func colorByThreshold(text string, value, warn, crit float64) string { switch { case value >= crit: return critStyle.Render(text) case value >= warn: return warnStyle.Render(text) default: return okStyle.Render(text) } }