chore: publish pxmon v0.2.0
This commit is contained in:
@@ -0,0 +1,497 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"pxmon/internal/cluster"
|
||||
"pxmon/internal/history"
|
||||
)
|
||||
|
||||
// liveClusterStat is one row on the live dashboard. Fields are best-effort;
|
||||
// if the agent is unreachable we still render the row with an error so the
|
||||
// operator sees the node exists but is degraded.
|
||||
type liveClusterStat struct {
|
||||
ClusterID string
|
||||
Name string
|
||||
Host string
|
||||
Reachable bool
|
||||
Err string
|
||||
CPUPercent float64
|
||||
MemPercent float64
|
||||
MemUsed uint64
|
||||
MemTotal uint64
|
||||
DiskUsePct float64
|
||||
NetMbps float64 // current: last point from history
|
||||
P95Mbps float64 // primary iface, 24h window
|
||||
Uplink string
|
||||
HostName string
|
||||
}
|
||||
|
||||
type monitorLiveMsg struct {
|
||||
entries []liveClusterStat
|
||||
err error
|
||||
}
|
||||
|
||||
type liveAutoTickMsg struct{}
|
||||
|
||||
func liveAutoTickCmd() tea.Cmd {
|
||||
return tea.Tick(liveAutoTickInterval, func(_ time.Time) tea.Msg {
|
||||
return liveAutoTickMsg{}
|
||||
})
|
||||
}
|
||||
|
||||
const (
|
||||
livePageSize = 14
|
||||
liveFetchWorkers = 16
|
||||
liveAutoTickInterval = 5 * time.Second
|
||||
liveP95HistoryWindow = 24 * time.Hour
|
||||
liveFetchClusterTO = 5 * time.Second
|
||||
liveFetchOverallTO = 25 * time.Second
|
||||
)
|
||||
|
||||
// fetchLiveCmd gathers live stats for every cluster with an installed agent
|
||||
// in parallel, pairs each with P95 from history, and emits a sorted slice.
|
||||
func (m monitorModel) fetchLiveCmd() tea.Cmd {
|
||||
svc := m.svc
|
||||
return func() tea.Msg {
|
||||
clusters, _, err := svc.List()
|
||||
if err != nil {
|
||||
return monitorLiveMsg{err: err}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), liveFetchOverallTO)
|
||||
defer cancel()
|
||||
|
||||
type work struct {
|
||||
idx int
|
||||
c cluster.Cluster
|
||||
}
|
||||
ch := make(chan work, len(clusters))
|
||||
results := make([]liveClusterStat, len(clusters))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
workers := liveFetchWorkers
|
||||
if workers > len(clusters) {
|
||||
workers = len(clusters)
|
||||
}
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for item := range ch {
|
||||
results[item.idx] = collectLiveRow(ctx, svc, item.c)
|
||||
}
|
||||
}()
|
||||
}
|
||||
for i, c := range clusters {
|
||||
ch <- work{idx: i, c: c}
|
||||
}
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
|
||||
out := make([]liveClusterStat, 0, len(results))
|
||||
for _, r := range results {
|
||||
if r.ClusterID == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return monitorLiveMsg{entries: out}
|
||||
}
|
||||
}
|
||||
|
||||
func collectLiveRow(ctx context.Context, svc *cluster.Service, c cluster.Cluster) liveClusterStat {
|
||||
row := liveClusterStat{
|
||||
ClusterID: c.ID,
|
||||
Name: c.Name,
|
||||
Host: c.Host,
|
||||
}
|
||||
if !c.Agent.Installed {
|
||||
row.Err = "agent not installed"
|
||||
return row
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, liveFetchClusterTO)
|
||||
defer cancel()
|
||||
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
|
||||
if err != nil {
|
||||
row.Err = err.Error()
|
||||
return row
|
||||
}
|
||||
row.Reachable = true
|
||||
row.CPUPercent = stats.CPU.UsagePercent
|
||||
row.MemPercent = stats.Memory.UsedPercent
|
||||
row.MemUsed = stats.Memory.UsedBytes
|
||||
row.MemTotal = stats.Memory.TotalBytes
|
||||
row.HostName = stats.Host.Hostname
|
||||
var maxDisk float64
|
||||
for _, d := range stats.Disk {
|
||||
if d.UsedPercent > maxDisk {
|
||||
maxDisk = d.UsedPercent
|
||||
}
|
||||
}
|
||||
row.DiskUsePct = maxDisk
|
||||
|
||||
// Historical P95 from the persisted store (populated by the sampler
|
||||
// running inside TUI or bot daemon).
|
||||
if store := svc.NetworkStore(); store != nil {
|
||||
since := time.Now().Add(-liveP95HistoryWindow)
|
||||
if snaps, err := store.Load(c.ID, since); err == nil && len(snaps) > 0 {
|
||||
primary := history.PrimaryInterface(snaps)
|
||||
row.Uplink = primary
|
||||
series := history.AggregateNodeSeries(snaps, primary)
|
||||
if len(series) > 0 {
|
||||
row.P95Mbps = history.PercentileMbps(series, 95)
|
||||
row.NetMbps = series[len(series)-1].TotalMbps
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback / live reading: take a second stats snapshot after a short
|
||||
// pause and derive the instantaneous per-interface Mbps ourselves.
|
||||
// This makes NET Mbps meaningful even when the history store is empty
|
||||
// (e.g. the sampler hasn't been running long).
|
||||
time.Sleep(700 * time.Millisecond)
|
||||
callCtx2, cancel2 := context.WithTimeout(ctx, liveFetchClusterTO)
|
||||
defer cancel2()
|
||||
stats2, err := svc.AgentStatsTyped(callCtx2, c.ID)
|
||||
if err != nil {
|
||||
return row
|
||||
}
|
||||
elapsed := stats2.Timestamp.Sub(stats.Timestamp).Seconds()
|
||||
if elapsed <= 0 {
|
||||
elapsed = 0.7
|
||||
}
|
||||
prev := make(map[string]struct {
|
||||
rx, tx uint64
|
||||
}, len(stats.Network))
|
||||
for _, n := range stats.Network {
|
||||
prev[n.Interface] = struct{ rx, tx uint64 }{n.RxBytes, n.TxBytes}
|
||||
}
|
||||
bestName := ""
|
||||
bestMbps := -1.0
|
||||
for _, n := range stats2.Network {
|
||||
if isLiveVirtual(n.Interface) {
|
||||
continue
|
||||
}
|
||||
p, ok := prev[n.Interface]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var rxB, txB uint64
|
||||
if n.RxBytes >= p.rx {
|
||||
rxB = n.RxBytes - p.rx
|
||||
}
|
||||
if n.TxBytes >= p.tx {
|
||||
txB = n.TxBytes - p.tx
|
||||
}
|
||||
rxMbps := float64(rxB) * 8 / elapsed / 1_000_000
|
||||
txMbps := float64(txB) * 8 / elapsed / 1_000_000
|
||||
m := rxMbps
|
||||
if txMbps > m {
|
||||
m = txMbps
|
||||
}
|
||||
if m > bestMbps {
|
||||
bestMbps = m
|
||||
bestName = n.Interface
|
||||
}
|
||||
}
|
||||
if bestMbps >= 0 {
|
||||
row.NetMbps = bestMbps
|
||||
if row.Uplink == "" {
|
||||
row.Uplink = bestName
|
||||
}
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
// isLiveVirtual mirrors history.isVirtualIface but is local so we don't
|
||||
// export the original.
|
||||
func isLiveVirtual(name string) bool {
|
||||
n := strings.ToLower(name)
|
||||
if n == "" || n == "lo" {
|
||||
return true
|
||||
}
|
||||
prefixes := []string{
|
||||
"lo", "docker", "br-", "br", "veth", "vnet", "tap", "virbr",
|
||||
"cni", "flannel", "wg", "tun", "tailscale", "zt", "kube",
|
||||
"cilium", "ovs", "podman", "dummy",
|
||||
}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(n, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// handleLiveKey routes keys while the live dashboard is visible.
|
||||
func (m monitorModel) handleLiveKey(v tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
switch v.String() {
|
||||
case "o", "esc":
|
||||
m.view = viewOverview
|
||||
return m, nil
|
||||
case "c":
|
||||
m.view = viewClusters
|
||||
return m, nil
|
||||
case "r", "R":
|
||||
m.liveLoading = true
|
||||
m.setStatus("live: refreshing")
|
||||
return m, m.fetchLiveCmd()
|
||||
case "up", "k":
|
||||
if m.liveCursor > 0 {
|
||||
m.liveCursor--
|
||||
}
|
||||
return m, nil
|
||||
case "down", "j":
|
||||
if m.liveCursor < len(m.liveEntries)-1 {
|
||||
m.liveCursor++
|
||||
}
|
||||
return m, nil
|
||||
case "pgup":
|
||||
if m.livePage > 0 {
|
||||
m.livePage--
|
||||
}
|
||||
return m, nil
|
||||
case "pgdown":
|
||||
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
|
||||
if m.livePage < pages-1 {
|
||||
m.livePage++
|
||||
}
|
||||
return m, nil
|
||||
case "home":
|
||||
m.liveCursor = 0
|
||||
m.livePage = 0
|
||||
return m, nil
|
||||
case "end":
|
||||
m.liveCursor = len(m.liveEntries) - 1
|
||||
if m.liveCursor < 0 {
|
||||
m.liveCursor = 0
|
||||
}
|
||||
pages := (len(m.liveEntries) + livePageSize - 1) / livePageSize
|
||||
if pages > 0 {
|
||||
m.livePage = pages - 1
|
||||
}
|
||||
return m, nil
|
||||
case "enter":
|
||||
if m.livePinned == nil {
|
||||
m.livePinned = map[string]bool{}
|
||||
}
|
||||
rows := m.sortedLiveEntries()
|
||||
if m.liveCursor >= 0 && m.liveCursor < len(rows) {
|
||||
id := rows[m.liveCursor].ClusterID
|
||||
if m.livePinned[id] {
|
||||
delete(m.livePinned, id)
|
||||
m.setStatus("live: unpinned " + rows[m.liveCursor].Name)
|
||||
} else {
|
||||
m.livePinned[id] = true
|
||||
m.setStatus("live: pinned " + rows[m.liveCursor].Name)
|
||||
}
|
||||
}
|
||||
return m, nil
|
||||
case "1":
|
||||
m.liveSort = 0
|
||||
return m, nil
|
||||
case "2":
|
||||
m.liveSort = 1
|
||||
return m, nil
|
||||
case "3":
|
||||
m.liveSort = 2
|
||||
return m, nil
|
||||
case "4":
|
||||
m.liveSort = 3
|
||||
return m, nil
|
||||
case "5":
|
||||
m.liveSort = 4
|
||||
return m, nil
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// sortedLiveEntries returns entries sorted by the active metric. Pinned
|
||||
// clusters always float to the top so the operator keeps eyes on them even
|
||||
// when their usage changes.
|
||||
func (m monitorModel) sortedLiveEntries() []liveClusterStat {
|
||||
entries := make([]liveClusterStat, len(m.liveEntries))
|
||||
copy(entries, m.liveEntries)
|
||||
key := func(r liveClusterStat) float64 {
|
||||
switch m.liveSort {
|
||||
case 1:
|
||||
return r.MemPercent
|
||||
case 2:
|
||||
return r.NetMbps
|
||||
case 3:
|
||||
return r.P95Mbps
|
||||
case 4:
|
||||
return r.DiskUsePct
|
||||
default:
|
||||
return r.CPUPercent
|
||||
}
|
||||
}
|
||||
sort.SliceStable(entries, func(i, j int) bool {
|
||||
pi := m.livePinned[entries[i].ClusterID]
|
||||
pj := m.livePinned[entries[j].ClusterID]
|
||||
if pi != pj {
|
||||
return pi
|
||||
}
|
||||
return key(entries[i]) > key(entries[j])
|
||||
})
|
||||
return entries
|
||||
}
|
||||
|
||||
func liveSortLabel(n int) string {
|
||||
switch n {
|
||||
case 1:
|
||||
return "ram"
|
||||
case 2:
|
||||
return "net"
|
||||
case 3:
|
||||
return "p95"
|
||||
case 4:
|
||||
return "disk"
|
||||
default:
|
||||
return "cpu"
|
||||
}
|
||||
}
|
||||
|
||||
// renderLiveDashboard paints the paginated live view.
|
||||
func (m monitorModel) renderLiveDashboard(width int) string {
|
||||
title := titleStyle.Render("━━ Live cluster fleet ━━")
|
||||
help := dimStyle.Render("sort: 1=cpu 2=ram 3=net 4=p95 5=disk enter pin r refresh ↑/↓ select pgup/pgdn page o back")
|
||||
|
||||
if m.liveLoading && len(m.liveEntries) == 0 {
|
||||
body := dimStyle.Render("collecting stats from every cluster…")
|
||||
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
|
||||
}
|
||||
if m.liveError != "" && len(m.liveEntries) == 0 {
|
||||
body := critStyle.Render("error: " + m.liveError)
|
||||
return lipgloss.JoinVertical(lipgloss.Left, title, help, "", body)
|
||||
}
|
||||
if len(m.liveEntries) == 0 {
|
||||
return lipgloss.JoinVertical(lipgloss.Left, title, help, "",
|
||||
dimStyle.Render("no clusters with installed agents"))
|
||||
}
|
||||
|
||||
entries := m.sortedLiveEntries()
|
||||
total := len(entries)
|
||||
pages := (total + livePageSize - 1) / livePageSize
|
||||
if pages == 0 {
|
||||
pages = 1
|
||||
}
|
||||
if m.livePage >= pages {
|
||||
m.livePage = pages - 1
|
||||
}
|
||||
start := m.livePage * livePageSize
|
||||
end := start + livePageSize
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
page := entries[start:end]
|
||||
|
||||
headerLine := fmt.Sprintf(" %-3s %-16s %-3s %6s %6s %14s %12s %12s %6s %-10s",
|
||||
"#", "NAME", "PIN", "CPU%", "RAM%", "RAM", "NET", "P95", "DISK%", "UPLINK")
|
||||
lines := []string{title, help, "",
|
||||
dimStyle.Render(fmt.Sprintf("sorted by %s · page %d/%d · %d clusters · refresh every %s · enter=pin",
|
||||
liveSortLabel(m.liveSort), m.livePage+1, pages, total, liveAutoTickInterval)),
|
||||
"",
|
||||
accentStyle.Bold(true).Render(headerLine),
|
||||
}
|
||||
|
||||
for i, row := range page {
|
||||
absoluteIdx := start + i
|
||||
selected := absoluteIdx == m.liveCursor
|
||||
pin := " "
|
||||
if m.livePinned[row.ClusterID] {
|
||||
pin = "⚑"
|
||||
}
|
||||
name := row.Name
|
||||
if len(name) > 16 {
|
||||
name = name[:13] + "..."
|
||||
}
|
||||
if !row.Reachable {
|
||||
line := fmt.Sprintf(" %-3d %-16s %-3s %s",
|
||||
absoluteIdx+1, name, pin, critStyle.Render("unreachable: "+clipRight(row.Err, 60)))
|
||||
if selected {
|
||||
line = selectedRowStyle.Render(line)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
continue
|
||||
}
|
||||
|
||||
cpuStr := colorByThreshold(fmt.Sprintf("%5.1f", row.CPUPercent), row.CPUPercent, 50, 80)
|
||||
ramStr := colorByThreshold(fmt.Sprintf("%5.1f", row.MemPercent), row.MemPercent, 50, 80)
|
||||
ramAbs := fmt.Sprintf("%7s/%-6s", humanBytesUint(row.MemUsed), humanBytesUint(row.MemTotal))
|
||||
netStr := fmt.Sprintf("%12s", formatMbpsHuman(row.NetMbps))
|
||||
var p95Str string
|
||||
if row.P95Mbps > 0 {
|
||||
p95Str = fmt.Sprintf("%12s", formatMbpsHuman(row.P95Mbps))
|
||||
} else {
|
||||
p95Str = dimStyle.Render(fmt.Sprintf("%12s", "— (no hist)"))
|
||||
}
|
||||
diskStr := colorByThreshold(fmt.Sprintf("%5.1f", row.DiskUsePct), row.DiskUsePct, 70, 90)
|
||||
uplink := row.Uplink
|
||||
if uplink == "" {
|
||||
uplink = "-"
|
||||
}
|
||||
if len(uplink) > 10 {
|
||||
uplink = uplink[:10]
|
||||
}
|
||||
|
||||
body := fmt.Sprintf(" %-3d %-16s %-3s %s %s %14s %s %s %s %-10s",
|
||||
absoluteIdx+1, name, pin,
|
||||
cpuStr, ramStr, ramAbs, netStr, p95Str, diskStr, uplink)
|
||||
if selected {
|
||||
body = selectedRowStyle.Render(body)
|
||||
}
|
||||
lines = append(lines, body)
|
||||
}
|
||||
|
||||
// Stats footer.
|
||||
var totalCPU, totalMem, totalNet, totalP95 float64
|
||||
reachable := 0
|
||||
for _, r := range entries {
|
||||
if !r.Reachable {
|
||||
continue
|
||||
}
|
||||
reachable++
|
||||
totalCPU += r.CPUPercent
|
||||
totalMem += r.MemPercent
|
||||
totalNet += r.NetMbps
|
||||
totalP95 += r.P95Mbps
|
||||
}
|
||||
if reachable > 0 {
|
||||
lines = append(lines, "",
|
||||
dimStyle.Render(fmt.Sprintf("fleet avg: cpu %.1f%% · ram %.1f%% · net %s · p95 %s · reachable %d/%d",
|
||||
totalCPU/float64(reachable),
|
||||
totalMem/float64(reachable),
|
||||
formatMbpsHuman(totalNet),
|
||||
formatMbpsHuman(totalP95),
|
||||
reachable, total)))
|
||||
}
|
||||
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
var selectedRowStyle = lipgloss.NewStyle().Background(lipgloss.Color("#3A3A3A")).Bold(true)
|
||||
|
||||
func colorByThreshold(text string, value, warn, crit float64) string {
|
||||
switch {
|
||||
case value >= crit:
|
||||
return critStyle.Render(text)
|
||||
case value >= warn:
|
||||
return warnStyle.Render(text)
|
||||
default:
|
||||
return okStyle.Render(text)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user