273 lines
7.5 KiB
Go
273 lines
7.5 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"time"
|
|
|
|
"pxmon/internal/agent"
|
|
"pxmon/internal/history"
|
|
)
|
|
|
|
// UsageRange is a façade over history.RangeShortcut so callers outside this
|
|
// package don't need to import both.
|
|
type UsageRange = history.RangeShortcut
|
|
|
|
// UsageSnapshot bundles everything the "usage" page shows for a cluster at a
|
|
// single point in time.
|
|
type UsageSnapshot struct {
|
|
ClusterID string `json:"cluster_id"`
|
|
ClusterName string `json:"cluster_name"`
|
|
Range string `json:"range"`
|
|
GeneratedAt time.Time `json:"generated_at"`
|
|
Live agent.StatsResponse `json:"live"`
|
|
Top agent.TopResponse `json:"top"`
|
|
DU agent.DUResponse `json:"du"`
|
|
NodeSeries []history.NodeSamplePoint `json:"series,omitempty"`
|
|
P95TotalMbps float64 `json:"p95_total_mbps"`
|
|
MaxTotalMbps float64 `json:"max_total_mbps"`
|
|
AvgTotalMbps float64 `json:"avg_total_mbps"`
|
|
TopIfaceName string `json:"top_iface_name,omitempty"`
|
|
TopIfaceMbps float64 `json:"top_iface_mbps,omitempty"`
|
|
DUError string `json:"du_error,omitempty"`
|
|
TopError string `json:"top_error,omitempty"`
|
|
HistoryError string `json:"history_error,omitempty"`
|
|
}
|
|
|
|
// AgentTopProcesses calls /api/v1/top on the selected cluster's agent.
|
|
func (s *Service) AgentTopProcesses(ctx context.Context, selector string, sampleWindow time.Duration, limit int) (agent.TopResponse, error) {
|
|
cluster, err := s.Get(selector)
|
|
if err != nil {
|
|
return agent.TopResponse{}, err
|
|
}
|
|
if !cluster.Agent.Installed {
|
|
return agent.TopResponse{}, errors.New("agent is not installed on this cluster")
|
|
}
|
|
|
|
ac, err := s.newAgentClient(ctx, cluster, 15*time.Second)
|
|
if err != nil {
|
|
return agent.TopResponse{}, err
|
|
}
|
|
defer ac.Close()
|
|
|
|
url := ac.target + "/api/v1/top"
|
|
q := ""
|
|
if sampleWindow > 0 {
|
|
q += "sample_ms=" + strconv.FormatInt(sampleWindow.Milliseconds(), 10)
|
|
}
|
|
if limit > 0 {
|
|
if q != "" {
|
|
q += "&"
|
|
}
|
|
q += "limit=" + strconv.Itoa(limit)
|
|
}
|
|
if q != "" {
|
|
url += "?" + q
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return agent.TopResponse{}, err
|
|
}
|
|
applyAgentRequestAuth(req, cluster)
|
|
|
|
resp, err := ac.http.Do(req)
|
|
if err != nil {
|
|
if normalizeTransport(cluster.Transport) == TransportIPFabric {
|
|
s.CloseTunnelClient(cluster.ID)
|
|
}
|
|
return agent.TopResponse{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return agent.TopResponse{}, err
|
|
}
|
|
if resp.StatusCode >= 300 {
|
|
msg := string(body)
|
|
if len(msg) > 200 {
|
|
msg = msg[:200]
|
|
}
|
|
return agent.TopResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
|
|
}
|
|
|
|
var out agent.TopResponse
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return agent.TopResponse{}, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// AgentDirSizes calls /api/v1/du on the selected cluster's agent.
|
|
func (s *Service) AgentDirSizes(ctx context.Context, selector, path string, limit int, timeout time.Duration) (agent.DUResponse, error) {
|
|
cluster, err := s.Get(selector)
|
|
if err != nil {
|
|
return agent.DUResponse{}, err
|
|
}
|
|
if !cluster.Agent.Installed {
|
|
return agent.DUResponse{}, errors.New("agent is not installed on this cluster")
|
|
}
|
|
|
|
httpTimeout := timeout + 10*time.Second
|
|
if httpTimeout < 20*time.Second {
|
|
httpTimeout = 20 * time.Second
|
|
}
|
|
ac, err := s.newAgentClient(ctx, cluster, httpTimeout)
|
|
if err != nil {
|
|
return agent.DUResponse{}, err
|
|
}
|
|
defer ac.Close()
|
|
|
|
url := ac.target + "/api/v1/du"
|
|
q := ""
|
|
if path != "" {
|
|
q += "path=" + path
|
|
}
|
|
if limit > 0 {
|
|
if q != "" {
|
|
q += "&"
|
|
}
|
|
q += "limit=" + strconv.Itoa(limit)
|
|
}
|
|
if timeout > 0 {
|
|
if q != "" {
|
|
q += "&"
|
|
}
|
|
q += "timeout_ms=" + strconv.FormatInt(timeout.Milliseconds(), 10)
|
|
}
|
|
if q != "" {
|
|
url += "?" + q
|
|
}
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return agent.DUResponse{}, err
|
|
}
|
|
applyAgentRequestAuth(req, cluster)
|
|
|
|
resp, err := ac.http.Do(req)
|
|
if err != nil {
|
|
if normalizeTransport(cluster.Transport) == TransportIPFabric {
|
|
s.CloseTunnelClient(cluster.ID)
|
|
}
|
|
return agent.DUResponse{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return agent.DUResponse{}, err
|
|
}
|
|
if resp.StatusCode >= 300 {
|
|
msg := string(body)
|
|
if len(msg) > 200 {
|
|
msg = msg[:200]
|
|
}
|
|
return agent.DUResponse{}, fmt.Errorf("agent HTTP %d: %s", resp.StatusCode, msg)
|
|
}
|
|
|
|
var out agent.DUResponse
|
|
if err := json.Unmarshal(body, &out); err != nil {
|
|
return agent.DUResponse{}, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// CollectUsageSnapshot gathers every data source needed by the usage page:
|
|
// live stats, top processes, top folders, and historical aggregate for P95.
|
|
// Errors in individual components are recorded on the snapshot instead of
|
|
// aborting the whole call — the page degrades gracefully.
|
|
func (s *Service) CollectUsageSnapshot(ctx context.Context, selector string, rng UsageRange, duPath string) (UsageSnapshot, error) {
|
|
cluster, err := s.Get(selector)
|
|
if err != nil {
|
|
return UsageSnapshot{}, err
|
|
}
|
|
|
|
snap := UsageSnapshot{
|
|
ClusterID: cluster.ID,
|
|
ClusterName: cluster.Name,
|
|
Range: string(rng),
|
|
GeneratedAt: time.Now().UTC(),
|
|
}
|
|
|
|
if live, err := s.AgentStatsTyped(ctx, selector); err != nil {
|
|
return snap, fmt.Errorf("live stats: %w", err)
|
|
} else {
|
|
snap.Live = live
|
|
}
|
|
|
|
if top, err := s.AgentTopProcesses(ctx, selector, 300*time.Millisecond, 15); err != nil {
|
|
snap.TopError = err.Error()
|
|
} else {
|
|
snap.Top = top
|
|
}
|
|
|
|
if duPath == "" {
|
|
duPath = "/"
|
|
}
|
|
if du, err := s.AgentDirSizes(ctx, selector, duPath, 12, 12*time.Second); err != nil {
|
|
snap.DUError = err.Error()
|
|
} else {
|
|
snap.DU = du
|
|
}
|
|
|
|
// Network history + P95 aggregation.
|
|
if s.networkStore == nil {
|
|
snap.HistoryError = "history store not configured"
|
|
} else {
|
|
since := history.RangeShortcut(rng).Since(time.Now())
|
|
if rng == "" || rng == history.RangeLive {
|
|
since = time.Now().Add(-5 * time.Minute)
|
|
}
|
|
snapshots, err := s.networkStore.Load(cluster.ID, since)
|
|
if err != nil {
|
|
snap.HistoryError = err.Error()
|
|
} else {
|
|
// Pick the physical uplink once and use it consistently for
|
|
// the series, P95, and the "top iface" display so they all
|
|
// describe the same thing.
|
|
primary := history.PrimaryInterface(snapshots)
|
|
series := history.AggregateNodeSeries(snapshots, primary)
|
|
snap.NodeSeries = series
|
|
snap.P95TotalMbps = history.PercentileMbps(series, 95)
|
|
if len(series) > 0 {
|
|
maxV := 0.0
|
|
sum := 0.0
|
|
for _, p := range series {
|
|
if p.TotalMbps > maxV {
|
|
maxV = p.TotalMbps
|
|
}
|
|
sum += p.TotalMbps
|
|
}
|
|
snap.MaxTotalMbps = maxV
|
|
snap.AvgTotalMbps = sum / float64(len(series))
|
|
}
|
|
snap.TopIfaceName = primary
|
|
snap.TopIfaceMbps = snap.AvgTotalMbps
|
|
}
|
|
}
|
|
|
|
return snap, nil
|
|
}
|
|
|
|
// RenderUsageChartPNG is a thin helper that reads a usage snapshot's series
|
|
// and delegates to history.RenderNodeNetworkPNG.
|
|
func RenderUsageChartPNG(snap UsageSnapshot, title string) ([]byte, error) {
|
|
if title == "" {
|
|
title = fmt.Sprintf("%s — node network usage (%s)", snap.ClusterName, snap.Range)
|
|
}
|
|
subtitle := fmt.Sprintf("P95 %.1f Mbps | max %.1f Mbps | avg %.1f Mbps",
|
|
snap.P95TotalMbps, snap.MaxTotalMbps, snap.AvgTotalMbps)
|
|
return history.RenderNodeNetworkPNG(snap.NodeSeries, history.ChartOptions{
|
|
Title: title,
|
|
Subtitle: subtitle,
|
|
Percentile: 95,
|
|
})
|
|
}
|