441 lines
13 KiB
Go
441 lines
13 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"mime/multipart"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"pxmon/internal/cluster"
|
|
"pxmon/internal/history"
|
|
)
|
|
|
|
type samplerIfaceCounter struct {
|
|
RxBytes uint64
|
|
TxBytes uint64
|
|
}
|
|
|
|
type samplerState struct {
|
|
Timestamp time.Time
|
|
Counters map[string]samplerIfaceCounter
|
|
}
|
|
|
|
var (
|
|
samplerMu sync.Mutex
|
|
samplerPrev = map[string]*samplerState{}
|
|
)
|
|
|
|
// handleBotUsageCommand intercepts usage/traffic/graph commands before they
|
|
// reach the generic runObserverScopedCommand bridge so it can send richer
|
|
// payloads (including PNG attachments) back to Telegram. Returns true when
|
|
// the command was consumed.
|
|
func handleBotUsageCommand(ctx context.Context, client *http.Client, svc *cluster.Service, token string, chatID int64, cmd string) bool {
|
|
fields := strings.Fields(cmd)
|
|
if len(fields) == 0 {
|
|
return false
|
|
}
|
|
rest := fields
|
|
if strings.EqualFold(fields[0], "cluster") && len(fields) > 1 {
|
|
rest = fields[1:]
|
|
}
|
|
head := strings.ToLower(rest[0])
|
|
if head != "usage" && head != "traffic" && head != "graph" && head != "p95" {
|
|
return false
|
|
}
|
|
|
|
selector := ""
|
|
rangeStr := defaultBotRange(head)
|
|
iface := ""
|
|
for _, f := range rest[1:] {
|
|
if strings.HasPrefix(f, "--iface=") {
|
|
iface = strings.TrimSpace(strings.TrimPrefix(f, "--iface="))
|
|
continue
|
|
}
|
|
if strings.HasPrefix(f, "--range=") {
|
|
rangeStr = strings.TrimPrefix(f, "--range=")
|
|
continue
|
|
}
|
|
if rng, ok := history.ParseRangeShortcut(f); ok {
|
|
rangeStr = string(rng)
|
|
continue
|
|
}
|
|
if selector == "" {
|
|
if head == "p95" && iface == "" && !strings.HasPrefix(f, "-") {
|
|
iface = f
|
|
} else {
|
|
selector = f
|
|
}
|
|
}
|
|
}
|
|
|
|
rng, ok := history.ParseRangeShortcut(strings.TrimSpace(rangeStr))
|
|
if !ok {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>%s</b>\n<pre>invalid range %q</pre>", head, rangeStr))
|
|
return true
|
|
}
|
|
|
|
gather, cancel := context.WithTimeout(ctx, 50*time.Second)
|
|
defer cancel()
|
|
|
|
snap, err := svc.CollectUsageSnapshot(gather, selector, rng, "/")
|
|
if err != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>%s</b>\n<pre>%s</pre>", head, htmlEscapeTelegram(err.Error())))
|
|
return true
|
|
}
|
|
|
|
switch head {
|
|
case "usage":
|
|
body := formatUsageForBot(snap, rng)
|
|
header := fmt.Sprintf("🟢 <b>usage %s</b> <i>%s</i>", htmlEscapeTelegram(snap.ClusterName), rng.Label())
|
|
for _, chunk := range splitTelegramText(htmlEscapeTelegram(body), 3400) {
|
|
_ = telegramSendHTML(ctx, client, token, chatID, header+"\n<pre>"+chunk+"</pre>")
|
|
header = ""
|
|
}
|
|
case "traffic":
|
|
body := formatTrafficForBot(snap, rng)
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🟢 <b>traffic %s</b> <i>%s</i>\n<pre>%s</pre>",
|
|
htmlEscapeTelegram(snap.ClusterName), rng.Label(), htmlEscapeTelegram(body)))
|
|
case "graph":
|
|
png, err := cluster.RenderUsageChartPNG(snap, "")
|
|
if err != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>graph</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
|
|
return true
|
|
}
|
|
caption := fmt.Sprintf("📈 <b>%s</b> <i>%s</i>\nP95: %s · max: %s · avg: %s · %d samples",
|
|
htmlEscapeTelegram(snap.ClusterName),
|
|
rng.Label(),
|
|
formatMbpsHuman(snap.P95TotalMbps),
|
|
formatMbpsHuman(snap.MaxTotalMbps),
|
|
formatMbpsHuman(snap.AvgTotalMbps),
|
|
len(snap.NodeSeries),
|
|
)
|
|
filename := sanitizeFilename(snap.ClusterName) + "-" + string(rng) + ".png"
|
|
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>graph</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
|
|
}
|
|
case "p95":
|
|
if strings.TrimSpace(iface) == "" {
|
|
_ = telegramSendHTML(ctx, client, token, chatID, "🔴 <b>p95</b>\n<pre>iface is required (example: p95 eth0 30d)</pre>")
|
|
return true
|
|
}
|
|
p95Snap, err := svc.CollectInterfaceP95(selector, iface, rng)
|
|
if err != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>p95</b>\n<pre>%s</pre>", htmlEscapeTelegram(err.Error())))
|
|
return true
|
|
}
|
|
png, err := svc.RenderInterfaceP95GraphPNG(p95Snap)
|
|
if err != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>p95</b>\n<pre>render failed: %s</pre>", htmlEscapeTelegram(err.Error())))
|
|
return true
|
|
}
|
|
filename := sanitizeFilename(p95Snap.ClusterName) + "-" + sanitizeFilename(p95Snap.Interface) + "-" + string(rng) + ".png"
|
|
caption := fmt.Sprintf("📈 <b>P95 %s</b> <i>%s</i>\niface: %s\nP95: %s · max: %s · avg: %s · %d samples",
|
|
htmlEscapeTelegram(p95Snap.ClusterName),
|
|
rng.Label(),
|
|
htmlEscapeTelegram(p95Snap.Interface),
|
|
formatMbpsHuman(p95Snap.P95Mbps),
|
|
formatMbpsHuman(p95Snap.MaxMbps),
|
|
formatMbpsHuman(p95Snap.AvgMbps),
|
|
p95Snap.Samples,
|
|
)
|
|
if sendErr := telegramSendPhoto(ctx, client, token, chatID, filename, png, caption); sendErr != nil {
|
|
_ = telegramSendHTML(ctx, client, token, chatID,
|
|
fmt.Sprintf("🔴 <b>p95</b>\n<pre>send failed: %s</pre>", htmlEscapeTelegram(sendErr.Error())))
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func defaultBotRange(cmd string) string {
|
|
switch cmd {
|
|
case "usage":
|
|
return "live"
|
|
case "traffic":
|
|
return "1h"
|
|
case "graph":
|
|
return "1d"
|
|
case "p95":
|
|
return "30d"
|
|
}
|
|
return "live"
|
|
}
|
|
|
|
// trySendGraphAttachmentFromCLIOutput is a safety net for graph commands that
|
|
// were executed through the generic CLI bridge and returned text like:
|
|
//
|
|
// Saved: <file.png>
|
|
//
|
|
// It uploads the generated PNG so Telegram users always receive the image.
|
|
func trySendGraphAttachmentFromCLIOutput(ctx context.Context, client *http.Client, token string, chatID int64, cmd, out string) (bool, error) {
|
|
fields := strings.Fields(strings.TrimSpace(cmd))
|
|
if len(fields) == 0 {
|
|
return false, nil
|
|
}
|
|
|
|
isGraph := false
|
|
if strings.EqualFold(fields[0], "graph") {
|
|
isGraph = true
|
|
}
|
|
if len(fields) >= 2 && strings.EqualFold(fields[0], "cluster") && strings.EqualFold(fields[1], "graph") {
|
|
isGraph = true
|
|
}
|
|
if !isGraph {
|
|
return false, nil
|
|
}
|
|
|
|
lines := strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n")
|
|
saved := ""
|
|
p95 := ""
|
|
for _, line := range lines {
|
|
t := strings.TrimSpace(stripANSI(line))
|
|
if strings.HasPrefix(strings.ToLower(t), "saved:") {
|
|
saved = strings.TrimSpace(t[len("saved:"):])
|
|
continue
|
|
}
|
|
if strings.HasPrefix(strings.ToLower(t), "p95:") {
|
|
p95 = t
|
|
}
|
|
}
|
|
if saved == "" {
|
|
return false, nil
|
|
}
|
|
|
|
data, err := os.ReadFile(saved)
|
|
if err != nil {
|
|
return false, fmt.Errorf("read graph png %q: %w", saved, err)
|
|
}
|
|
|
|
filename := filepath.Base(saved)
|
|
caption := "📈 <b>cluster graph</b>"
|
|
if p95 != "" {
|
|
caption += "\n" + htmlEscapeTelegram(p95)
|
|
}
|
|
if err := telegramSendPhoto(ctx, client, token, chatID, filename, data, caption); err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func formatTrafficForBot(snap cluster.UsageSnapshot, rng history.RangeShortcut) string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "range: %s\n", rng.Label())
|
|
fmt.Fprintf(&b, "samples: %d\n", len(snap.NodeSeries))
|
|
fmt.Fprintf(&b, "P95: %s\n", formatMbpsHuman(snap.P95TotalMbps))
|
|
fmt.Fprintf(&b, "max: %s\n", formatMbpsHuman(snap.MaxTotalMbps))
|
|
fmt.Fprintf(&b, "avg: %s\n", formatMbpsHuman(snap.AvgTotalMbps))
|
|
if snap.TopIfaceName != "" {
|
|
fmt.Fprintf(&b, "uplink: %s (avg %s)\n", snap.TopIfaceName, formatMbpsHuman(snap.TopIfaceMbps))
|
|
}
|
|
if snap.HistoryError != "" {
|
|
fmt.Fprintf(&b, "note: %s\n", snap.HistoryError)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// telegramSendPhoto uploads a PNG to Telegram via multipart sendPhoto.
|
|
func telegramSendPhoto(ctx context.Context, client *http.Client, token string, chatID int64, filename string, png []byte, captionHTML string) error {
|
|
var buf bytes.Buffer
|
|
w := multipart.NewWriter(&buf)
|
|
|
|
if err := w.WriteField("chat_id", strconv.FormatInt(chatID, 10)); err != nil {
|
|
return err
|
|
}
|
|
if captionHTML != "" {
|
|
if err := w.WriteField("caption", captionHTML); err != nil {
|
|
return err
|
|
}
|
|
if err := w.WriteField("parse_mode", "HTML"); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
part, err := w.CreateFormFile("photo", filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := part.Write(png); err != nil {
|
|
return err
|
|
}
|
|
if err := w.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
endpoint := "https://api.telegram.org/bot" + token + "/sendPhoto"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, &buf)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req.Header.Set("Content-Type", w.FormDataContentType())
|
|
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode >= 300 {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
msg := strings.TrimSpace(string(body))
|
|
if len(msg) > 200 {
|
|
msg = msg[:200]
|
|
}
|
|
return fmt.Errorf("sendPhoto HTTP %d: %s", resp.StatusCode, msg)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// runHistorySampler periodically appends a network snapshot for every
|
|
// cluster that has an agent installed so long-running non-TUI processes
|
|
// (like the bot daemon) can build up data for P95/graph queries.
|
|
func runHistorySampler(ctx context.Context, svc *cluster.Service, interval time.Duration, logErr io.Writer) {
|
|
if svc == nil || svc.NetworkStore() == nil {
|
|
return
|
|
}
|
|
if interval <= 0 {
|
|
interval = 30 * time.Second
|
|
}
|
|
|
|
ticker := time.NewTicker(interval)
|
|
defer ticker.Stop()
|
|
|
|
sampleOnce(ctx, svc, logErr)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
sampleOnce(ctx, svc, logErr)
|
|
}
|
|
}
|
|
}
|
|
|
|
func sampleOnce(ctx context.Context, svc *cluster.Service, logErr io.Writer) {
|
|
clusters, _, err := svc.List()
|
|
if err != nil {
|
|
return
|
|
}
|
|
store := svc.NetworkStore()
|
|
if store == nil {
|
|
return
|
|
}
|
|
availStore := history.NewAvailabilityStore(svc.DataDir())
|
|
capStore := history.NewCapacityStore(svc.DataDir())
|
|
for _, c := range clusters {
|
|
if !c.Agent.Installed {
|
|
continue
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, 6*time.Second)
|
|
stats, err := svc.AgentStatsTyped(callCtx, c.ID)
|
|
cancel()
|
|
if err != nil {
|
|
fmt.Fprintf(logErr, "history sampler: %s: %v\n", c.Name, err)
|
|
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
|
|
Timestamp: time.Now().UTC(),
|
|
ClusterUp: false,
|
|
Error: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
items := make([]history.InterfaceSample, 0, len(stats.Network))
|
|
// With a single stats point we don't have rate info; estimate Mbps
|
|
// per-interface as a simple counter delta versus the sampler's prior
|
|
// snapshot if available.
|
|
samplerMu.Lock()
|
|
prev := samplerPrev[c.ID]
|
|
samplerMu.Unlock()
|
|
now := time.Now().UTC()
|
|
for _, n := range stats.Network {
|
|
rxMbps := 0.0
|
|
txMbps := 0.0
|
|
if prev != nil {
|
|
if pn, ok := prev.Counters[n.Interface]; ok {
|
|
elapsed := now.Sub(prev.Timestamp).Seconds()
|
|
if elapsed > 0.1 {
|
|
if n.RxBytes >= pn.RxBytes {
|
|
rxMbps = float64(n.RxBytes-pn.RxBytes) * 8 / elapsed / 1_000_000
|
|
}
|
|
if n.TxBytes >= pn.TxBytes {
|
|
txMbps = float64(n.TxBytes-pn.TxBytes) * 8 / elapsed / 1_000_000
|
|
}
|
|
}
|
|
}
|
|
}
|
|
items = append(items, history.InterfaceSample{
|
|
Interface: n.Interface,
|
|
RxMbps: rxMbps,
|
|
TxMbps: txMbps,
|
|
RxDrops: n.RxDrops,
|
|
TxDrops: n.TxDrops,
|
|
})
|
|
}
|
|
counters := make(map[string]samplerIfaceCounter, len(stats.Network))
|
|
for _, n := range stats.Network {
|
|
counters[n.Interface] = samplerIfaceCounter{RxBytes: n.RxBytes, TxBytes: n.TxBytes}
|
|
}
|
|
samplerMu.Lock()
|
|
samplerPrev[c.ID] = &samplerState{Timestamp: now, Counters: counters}
|
|
samplerMu.Unlock()
|
|
|
|
// Only persist when we have a real delta — skip the first bootstrap
|
|
// sample per cluster to avoid a meaningless zero row.
|
|
if prev != nil {
|
|
snap := history.NetworkSnapshot{
|
|
Timestamp: now,
|
|
Interfaces: items,
|
|
}
|
|
if err := store.Append(c.ID, snap); err != nil {
|
|
fmt.Fprintf(logErr, "history sampler: append %s: %v\n", c.Name, err)
|
|
}
|
|
}
|
|
// Availability and VM state snapshot.
|
|
vmStates := map[string]string{}
|
|
vmCtx, vmCancel := context.WithTimeout(ctx, 6*time.Second)
|
|
if rawStates, vmErr := svc.ListVMStates(vmCtx, c.ID); vmErr == nil {
|
|
vmStates = rawStates
|
|
}
|
|
vmCancel()
|
|
_ = availStore.Append(c.ID, history.AvailabilitySnapshot{
|
|
Timestamp: time.Now().UTC(),
|
|
ClusterUp: true,
|
|
VMStates: vmStates,
|
|
})
|
|
// Capacity snapshot (disk usage trend source).
|
|
disks := make([]history.CapacityDiskPoint, 0, len(stats.Disk))
|
|
for _, d := range stats.Disk {
|
|
if d.TotalBytes == 0 {
|
|
continue
|
|
}
|
|
disks = append(disks, history.CapacityDiskPoint{
|
|
Mount: d.MountPoint,
|
|
UsedBytes: d.UsedBytes,
|
|
TotalBytes: d.TotalBytes,
|
|
})
|
|
}
|
|
if len(disks) > 0 {
|
|
_ = capStore.Append(c.ID, history.CapacitySnapshot{
|
|
Timestamp: time.Now().UTC(),
|
|
Disks: disks,
|
|
})
|
|
}
|
|
}
|
|
}
|