chore: publish pxmon v0.2.0
This commit is contained in:
@@ -0,0 +1,440 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"pxmon/internal/cluster"
|
||||
"pxmon/internal/history"
|
||||
)
|
||||
|
||||
type samplerIfaceCounter struct {
|
||||
RxBytes uint64
|
||||
TxBytes uint64
|
||||
}
|
||||
|
||||
type samplerState struct {
|
||||
Timestamp time.Time
|
||||
Counters map[string]samplerIfaceCounter
|
||||
}
|
||||
|
||||
var (
|
||||
samplerMu sync.Mutex
|
||||
samplerPrev = map[string]*samplerState{}
|
||||
)
|
||||
|
||||
// handleBotUsageCommand intercepts usage/traffic/graph commands before they
|
||||
// reach the generic runObserverScopedCommand bridge so it can send richer
|
||||
// payloads (including PNG attachments) back to Telegram. Returns true when
|
||||
// the command was consumed.
|
||||
func handleBotUsageCommand(ctx context.Context, client *http.Client, svc *cluster.Service, token string, chatID int64, cmd string) bool {
|
||||
fields := strings.Fields(cmd)
|
||||
if len(fields) == 0 {
|
||||
return false
|
||||
}
|
||||
rest := fields
|
||||
if strings.EqualFold(fields[0], "cluster") && len(fields) > 1 {
|
||||
rest = fields[1:]
|
||||
}
|
||||
head := strings.ToLower(rest[0])
|
||||
if head != "usage" && head != "traffic" && head != "graph" && head != "p95" {
|
||||
return false
|
||||
}
|
||||
|
||||
selector := ""
|
||||
rangeStr := defaultBotRange(head)
|
||||
iface := ""
|
||||
for _, f := range rest[1:] {
|
||||
if strings.HasPrefix(f, "--iface=") {
|
||||
iface = strings.TrimSpace(strings.TrimPrefix(f, "--iface="))
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(f, "--range=") {
|
||||
rangeStr = strings.TrimPrefix(f, "--range=")
|
||||
continue
|
||||
}
|
||||
if rng, ok := history.ParseRangeShortcut(f); ok {
|
||||
rangeStr = string(rng)
|
||||
continue
|
||||
}
|
||||
if selector == "" {
|
||||
if head == "p95" && iface == "" && !strings.HasPrefix(f, "-") {
|
||||
iface = f
|
||||
} else {
|
||||
selector = f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(rangeStr))
|
||||
if !ok {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>%s</b>\n<pre>invalid range %q</pre>", head, rangeStr))
|
||||
return true
|
||||
}
|
||||
|
||||
gather, cancel := context.WithTimeout(ctx, 50*time.Second)
|
||||
defer cancel()
|
||||
|
||||
snap, err := svc.CollectUsageSnapshot(gather, selector, rng, "/")
|
||||
if err != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>%s</b>\n<pre>%s</pre>", head, htmlEscapeTelegram(err.Error())))
|
||||
return true
|
||||
}
|
||||
|
||||
switch head {
|
||||
case "usage":
|
||||
body := formatUsageForBot(snap, rng)
|
||||
header := fmt.Sprintf("🟢 <b>usage %s</b> <i>%s</i>", htmlEscapeTelegram(snap.ClusterName), rng.Label())
|
||||
for _, chunk := range splitTelegramText(htmlEscapeTelegram(body), 3400) {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID, header+"\n<pre>"+chunk+"</pre>")
|
||||
header = ""
|
||||
}
|
||||
case "traffic":
|
||||
body := formatTrafficForBot(snap, rng)
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🟢 <b>traffic %s</b> <i>%s</i>\n<pre>%s</pre>",
|
||||
htmlEscapeTelegram(snap.ClusterName), rng.Label(), htmlEscapeTelegram(body)))
|
||||
case "graph":
|
||||
png, err := cluster.RenderUsageChartPNG(snap, "")
|
||||
if err != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>graph</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
|
||||
return true
|
||||
}
|
||||
caption := fmt.Sprintf("📈 <b>%s</b> <i>%s</i>\nP95: %s · max: %s · avg: %s · %d samples",
|
||||
htmlEscapeTelegram(snap.ClusterName),
|
||||
rng.Label(),
|
||||
formatMbpsHuman(snap.P95TotalMbps),
|
||||
formatMbpsHuman(snap.MaxTotalMbps),
|
||||
formatMbpsHuman(snap.AvgTotalMbps),
|
||||
len(snap.NodeSeries),
|
||||
)
|
||||
filename := sanitizeFilename(snap.ClusterName) + "-" + string(rng) + ".png"
|
||||
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>graph</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
|
||||
}
|
||||
case "p95":
|
||||
if strings.TrimSpace(iface) == "" {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID, "🔴 <b>p95</b>\n<pre>iface is required (example: p95 eth0 30d)</pre>")
|
||||
return true
|
||||
}
|
||||
p95Snap, err := svc.CollectInterfaceP95(selector, iface, rng)
|
||||
if err != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>p95</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
|
||||
return true
|
||||
}
|
||||
png, err := svc.RenderInterfaceP95GraphPNG(p95Snap)
|
||||
if err != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>p95</b>\n<pre>render failed: %s</pre>", htmlEscapeTelegram(err.Error())))
|
||||
return true
|
||||
}
|
||||
filename := sanitizeFilename(p95Snap.ClusterName) + "-" + sanitizeFilename(p95Snap.Interface) + "-" + string(rng) + ".png"
|
||||
caption := fmt.Sprintf("📈 <b>P95 %s</b> <i>%s</i>\niface: %s\nP95: %s · max: %s · avg: %s · %d samples",
|
||||
htmlEscapeTelegram(p95Snap.ClusterName),
|
||||
rng.Label(),
|
||||
htmlEscapeTelegram(p95Snap.Interface),
|
||||
formatMbpsHuman(p95Snap.P95Mbps),
|
||||
formatMbpsHuman(p95Snap.MaxMbps),
|
||||
formatMbpsHuman(p95Snap.AvgMbps),
|
||||
p95Snap.Samples,
|
||||
)
|
||||
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
|
||||
_ = telegramSendHTML(ctx, client, token, chatID,
|
||||
fmt.Sprintf("🔴 <b>p95</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func defaultBotRange(cmd string) string {
|
||||
switch cmd {
|
||||
case "usage":
|
||||
return "live"
|
||||
case "traffic":
|
||||
return "1h"
|
||||
case "graph":
|
||||
return "1d"
|
||||
case "p95":
|
||||
return "30d"
|
||||
}
|
||||
return "live"
|
||||
}
|
||||
|
||||
// trySendGraphAttachmentFromCLIOutput is a safety net for graph commands that
|
||||
// were executed through the generic CLI bridge and returned text like:
|
||||
//
|
||||
// Saved: <file.png>
|
||||
//
|
||||
// It uploads the generated PNG so Telegram users always receive the image.
|
||||
func trySendGraphAttachmentFromCLIOutput(ctx context.Context, client *http.Client, token string, chatID int64, cmd, out string) (bool, error) {
|
||||
fields := strings.Fields(strings.TrimSpace(cmd))
|
||||
if len(fields) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
isGraph := false
|
||||
if strings.EqualFold(fields[0], "graph") {
|
||||
isGraph = true
|
||||
}
|
||||
if len(fields) >= 2 && strings.EqualFold(fields[0], "cluster") && strings.EqualFold(fields[1], "graph") {
|
||||
isGraph = true
|
||||
}
|
||||
if !isGraph {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
lines := strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n")
|
||||
saved := ""
|
||||
p95 := ""
|
||||
for _, line := range lines {
|
||||
t := strings.TrimSpace(stripANSI(line))
|
||||
if strings.HasPrefix(strings.ToLower(t), "saved:") {
|
||||
saved = strings.TrimSpace(t[len("saved:"):])
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(t), "p95:") {
|
||||
p95 = t
|
||||
}
|
||||
}
|
||||
if saved == "" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(saved)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read graph png %q: %w", saved, err)
|
||||
}
|
||||
|
||||
filename := filepath.Base(saved)
|
||||
caption := "📈 <b>cluster graph</b>"
|
||||
if p95 != "" {
|
||||
caption += "\n" + htmlEscapeTelegram(p95)
|
||||
}
|
||||
if err := telegramSendPhoto(ctx, client, token, chatID, filename, data, caption); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func formatTrafficForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "range: %s\n", rng.Label())
|
||||
fmt.Fprintf(&b, "samples: %d\n", len(snap.NodeSeries))
|
||||
fmt.Fprintf(&b, "P95: %s\n", formatMbpsHuman(snap.P95TotalMbps))
|
||||
fmt.Fprintf(&b, "max: %s\n", formatMbpsHuman(snap.MaxTotalMbps))
|
||||
fmt.Fprintf(&b, "avg: %s\n", formatMbpsHuman(snap.AvgTotalMbps))
|
||||
if snap.TopIfaceName != "" {
|
||||
fmt.Fprintf(&b, "uplink: %s (avg %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps))
|
||||
}
|
||||
if snap.HistoryError != "" {
|
||||
fmt.Fprintf(&b, "note: %s\n", snap.HistoryError)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// telegramSendPhoto uploads a PNG to Telegram via multipart sendPhoto.
|
||||
func telegramSendPhoto(ctx context.Context, client *http.Client, token string, chatID int64, filename string, png []byte, captionHTML string) error {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
|
||||
if err := w.WriteField("chat_id", strconv.FormatInt(chatID, 10)); err != nil {
|
||||
return err
|
||||
}
|
||||
if captionHTML != "" {
|
||||
if err := w.WriteField("caption", captionHTML); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.WriteField("parse_mode", "HTML"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
part, err := w.CreateFormFile("photo", filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := part.Write(png); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
endpoint := "https://api.telegram.org/bot" + token + "/sendPhoto"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, &buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Content-Type", w.FormDataContentType())
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
msg := strings.TrimSpace(string(body))
|
||||
if len(msg) > 200 {
|
||||
msg = msg[:200]
|
||||
}
|
||||
return fmt.Errorf("sendPhoto HTTP %d: %s", resp.StatusCode, msg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// runHistorySampler periodically appends a network snapshot for every
|
||||
// cluster that has an agent installed so long-running non-TUI processes
|
||||
// (like the bot daemon) can build up data for P95/graph queries.
|
||||
func runHistorySampler(ctx context.Context, svc *cluster.Service, interval time.Duration, logErr io.Writer) {
|
||||
if svc == nil || svc.NetworkStore() == nil {
|
||||
return
|
||||
}
|
||||
if interval <= 0 {
|
||||
interval = 30 * time.Second
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
sampleOnce(ctx, svc, logErr)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
sampleOnce(ctx, svc, logErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sampleOnce(ctx context.Context, svc *cluster.Service, logErr io.Writer) {
|
||||
clusters, _, err := svc.List()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
store := svc.NetworkStore()
|
||||
if store == nil {
|
||||
return
|
||||
}
|
||||
availStore := history.NewAvailabilityStore(svc.DataDir())
|
||||
capStore := history.NewCapacityStore(svc.DataDir())
|
||||
for _, c := range clusters {
|
||||
if !c.Agent.Installed {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
callCtx, cancel := context.WithTimeout(ctx, 6*time.Second)
|
||||
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
|
||||
cancel()
|
||||
if err != nil {
|
||||
fmt.Fprintf(logErr, "history sampler: %s: %v\n", c.Name, err)
|
||||
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
|
||||
Timestamp: time.Now().UTC(),
|
||||
ClusterUp: false,
|
||||
Error: err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
items := make([]history.InterfaceSample, 0, len(stats.Network))
|
||||
// With a single stats point we don't have rate info; estimate Mbps
|
||||
// per-interface as a simple counter delta versus the sampler's prior
|
||||
// snapshot if available.
|
||||
samplerMu.Lock()
|
||||
prev := samplerPrev[c.ID]
|
||||
samplerMu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
for _, n := range stats.Network {
|
||||
rxMbps := 0.0
|
||||
txMbps := 0.0
|
||||
if prev != nil {
|
||||
if pn, ok := prev.Counters[n.Interface]; ok {
|
||||
elapsed := now.Sub(prev.Timestamp).Seconds()
|
||||
if elapsed > 0.1 {
|
||||
if n.RxBytes >= pn.RxBytes {
|
||||
rxMbps = float64(n.RxBytes-pn.RxBytes) * 8 / elapsed / 1_000_000
|
||||
}
|
||||
if n.TxBytes >= pn.TxBytes {
|
||||
txMbps = float64(n.TxBytes-pn.TxBytes) * 8 / elapsed / 1_000_000
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
items = append(items, history.InterfaceSample{
|
||||
Interface: n.Interface,
|
||||
RxMbps: rxMbps,
|
||||
TxMbps: txMbps,
|
||||
RxDrops: n.RxDrops,
|
||||
TxDrops: n.TxDrops,
|
||||
})
|
||||
}
|
||||
counters := make(map[string]samplerIfaceCounter, len(stats.Network))
|
||||
for _, n := range stats.Network {
|
||||
counters[n.Interface] = samplerIfaceCounter{RxBytes: n.RxBytes, TxBytes: n.TxBytes}
|
||||
}
|
||||
samplerMu.Lock()
|
||||
samplerPrev[c.ID] = &samplerState{Timestamp: now, Counters: counters}
|
||||
samplerMu.Unlock()
|
||||
|
||||
// Only persist when we have a real delta — skip the first bootstrap
|
||||
// sample per cluster to avoid a meaningless zero row.
|
||||
if prev != nil {
|
||||
snap := history.NetworkSnapshot{
|
||||
Timestamp: now,
|
||||
Interfaces: items,
|
||||
}
|
||||
if err := store.Append(c.ID, snap); err != nil {
|
||||
fmt.Fprintf(logErr, "history sampler: append %s: %v\n", c.Name, err)
|
||||
}
|
||||
}
|
||||
// Availability and VM state snapshot.
|
||||
vmStates := map[string]string{}
|
||||
vmCtx, vmCancel := context.WithTimeout(ctx, 6*time.Second)
|
||||
if rawStates, vmErr := svc.ListVMStates(vmCtx, c.ID); vmErr == nil {
|
||||
vmStates = rawStates
|
||||
}
|
||||
vmCancel()
|
||||
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
|
||||
Timestamp: time.Now().UTC(),
|
||||
ClusterUp: true,
|
||||
VMStates: vmStates,
|
||||
})
|
||||
// Capacity snapshot (disk usage trend source).
|
||||
disks := make([]history.CapacityDiskPoint, 0, len(stats.Disk))
|
||||
for _, d := range stats.Disk {
|
||||
if d.TotalBytes == 0 {
|
||||
continue
|
||||
}
|
||||
disks = append(disks, history.CapacityDiskPoint{
|
||||
Mount: d.MountPoint,
|
||||
UsedBytes: d.UsedBytes,
|
||||
TotalBytes: d.TotalBytes,
|
||||
})
|
||||
}
|
||||
if len(disks) > 0 {
|
||||
_ = capStore.Append(c.ID, history.CapacitySnapshot{
|
||||
Timestamp: time.Now().UTC(),
|
||||
Disks: disks,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user