mirror of
https://github.com/MHSanaei/3x-ui.git
synced 2026-09-02 16:37:14 +00:00
perf(metrics): tiered rollup history (7d at ~1.5MB) and cleaner ranges
Replace the flat 48h@2s ring buffer with a 3-tier rollup ladder (2s/1h, 1m/48h, 10m/7d). A sample feeds every tier and rolls up into progressively coarser averages, so per-metric footprint drops from ~21MB to ~1.5MB (measured, 16 system metrics) while extending the range from 48h to 7 days. aggregate() picks the finest tier covering the requested span; a pre-tier flat gob is migrated by replaying its samples through the rollup. Tidy the dashboard ranges to a professional ladder: 2m, 1h, 3h, 6h, 12h, 24h, 2d, 7d (drop the irregular 2h/5h, the redundant 30m, and the excessive 30d). The allow-list keeps bucket 30 because the node history panel uses it. Add an initial FreeOSMemory about 60s after boot to reclaim the startup and metric-restore peak instead of waiting for the periodic release. Cover the rollup, tier selection, round-trip, and footprint with tests.
This commit is contained in:
@@ -137,10 +137,13 @@ export default function SystemHistoryModal({ open, status, onClose }: SystemHist
|
|||||||
const tss: number[] = [];
|
const tss: number[] = [];
|
||||||
for (const p of msg.obj) {
|
for (const p of msg.obj) {
|
||||||
const d = new Date(p.t * 1000);
|
const d = new Date(p.t * 1000);
|
||||||
|
const MM = String(d.getMonth() + 1).padStart(2, '0');
|
||||||
|
const DD = String(d.getDate()).padStart(2, '0');
|
||||||
const hh = String(d.getHours()).padStart(2, '0');
|
const hh = String(d.getHours()).padStart(2, '0');
|
||||||
const mm = String(d.getMinutes()).padStart(2, '0');
|
const mm = String(d.getMinutes()).padStart(2, '0');
|
||||||
const ss = String(d.getSeconds()).padStart(2, '0');
|
const ss = String(d.getSeconds()).padStart(2, '0');
|
||||||
labs.push(bucket >= 60 ? `${hh}:${mm}` : `${hh}:${mm}:${ss}`);
|
const lab = bucket >= 2880 ? `${MM}-${DD} ${hh}:${mm}` : bucket >= 60 ? `${hh}:${mm}` : `${hh}:${mm}:${ss}`;
|
||||||
|
labs.push(lab);
|
||||||
vals.push(Number(p.v) || 0);
|
vals.push(Number(p.v) || 0);
|
||||||
tss.push(Number(p.t) || 0);
|
tss.push(Number(p.t) || 0);
|
||||||
}
|
}
|
||||||
@@ -208,14 +211,13 @@ export default function SystemHistoryModal({ open, status, onClose }: SystemHist
|
|||||||
onChange={setBucket}
|
onChange={setBucket}
|
||||||
options={[
|
options={[
|
||||||
{ value: 2, label: '2m' },
|
{ value: 2, label: '2m' },
|
||||||
{ value: 30, label: '30m' },
|
|
||||||
{ value: 60, label: '1h' },
|
{ value: 60, label: '1h' },
|
||||||
{ value: 120, label: '2h' },
|
|
||||||
{ value: 180, label: '3h' },
|
{ value: 180, label: '3h' },
|
||||||
{ value: 300, label: '5h' },
|
{ value: 360, label: '6h' },
|
||||||
{ value: 720, label: '12h' },
|
{ value: 720, label: '12h' },
|
||||||
{ value: 1440, label: '24h' },
|
{ value: 1440, label: '24h' },
|
||||||
{ value: 2880, label: '48h' },
|
{ value: 2880, label: '2d' },
|
||||||
|
{ value: 10080, label: '7d' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -286,11 +286,10 @@ export default function XrayMetricsModal({ open, onClose }: XrayMetricsModalProp
|
|||||||
onChange={setBucket}
|
onChange={setBucket}
|
||||||
options={[
|
options={[
|
||||||
{ value: 2, label: '2m' },
|
{ value: 2, label: '2m' },
|
||||||
{ value: 30, label: '30m' },
|
|
||||||
{ value: 60, label: '1h' },
|
{ value: 60, label: '1h' },
|
||||||
{ value: 120, label: '2h' },
|
|
||||||
{ value: 180, label: '3h' },
|
{ value: 180, label: '3h' },
|
||||||
{ value: 300, label: '5h' },
|
{ value: 360, label: '6h' },
|
||||||
|
{ value: 720, label: '12h' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"encoding/gob"
|
"encoding/gob"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -19,109 +18,167 @@ type MetricSample struct {
|
|||||||
V float64 `json:"v"`
|
V float64 `json:"v"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// metricCapacityDefault caps each ring buffer at 48h worth of @2s samples.
|
// tierSpec defines one resolution layer of the rollup ladder: a fixed bucket
|
||||||
// Node metrics arrive less frequently, so they fit the same retention window
|
// size in seconds and how many buckets to retain. window = resolution*capacity.
|
||||||
// with room to spare.
|
type tierSpec struct {
|
||||||
const metricCapacityDefault = 86400
|
resolution int
|
||||||
|
capacity int
|
||||||
// metricHistory is a thread-safe, in-memory ring buffer keyed by
|
|
||||||
// arbitrary strings. Two singletons live below: one for system-wide
|
|
||||||
// host metrics, one for per-node metrics. Keeping them in this file
|
|
||||||
// (rather than scattered across services) makes the storage model
|
|
||||||
// easy to reason about and avoids double-locking.
|
|
||||||
type metricHistory struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
metrics map[string][]MetricSample
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMetricHistory() *metricHistory {
|
// metricTiers is the rollup ladder applied to every series. High resolution is
|
||||||
return &metricHistory{metrics: map[string][]MetricSample{}}
|
// kept only for the recent past; older samples roll up into progressively
|
||||||
|
// coarser, cheaper layers (RRDtool-style). Per series this totals ~5700 samples
|
||||||
|
// (~90 KiB) yet spans a live 2s view through ~7 days of history.
|
||||||
|
var metricTiers = []tierSpec{
|
||||||
|
{resolution: 2, capacity: 1800}, // 1h at 2s
|
||||||
|
{resolution: 60, capacity: 2880}, // 48h at 1m
|
||||||
|
{resolution: 600, capacity: 1008}, // 7d at 10m
|
||||||
}
|
}
|
||||||
|
|
||||||
// append stores a single sample for the given metric, deduping when
|
// tierBuf is one fixed-resolution ring of a series. Samples land in an open
|
||||||
// two appends happen within the same wall-clock second (which can
|
// bucket and are averaged into the ring only when the next bucket begins, so a
|
||||||
// happen if the cron tick is faster than the metric's natural rate).
|
// coarse tier carries one mean per bucket instead of every raw point.
|
||||||
func (h *metricHistory) append(metric string, t time.Time, v float64) {
|
type tierBuf struct {
|
||||||
h.mu.Lock()
|
resolution int
|
||||||
defer h.mu.Unlock()
|
capacity int
|
||||||
buf := h.metrics[metric]
|
samples []MetricSample
|
||||||
p := MetricSample{T: t.Unix(), V: v}
|
open bool
|
||||||
if n := len(buf); n > 0 && buf[n-1].T == p.T {
|
openStart int64
|
||||||
buf[n-1] = p
|
openSum float64
|
||||||
} else {
|
openCount int
|
||||||
buf = append(buf, p)
|
}
|
||||||
|
|
||||||
|
func (tb *tierBuf) add(unixSec int64, v float64) {
|
||||||
|
res := int64(tb.resolution)
|
||||||
|
b := (unixSec / res) * res
|
||||||
|
if tb.open && b != tb.openStart {
|
||||||
|
tb.flush()
|
||||||
}
|
}
|
||||||
if len(buf) > metricCapacityDefault {
|
tb.open = true
|
||||||
buf = buf[len(buf)-metricCapacityDefault:]
|
tb.openStart = b
|
||||||
|
tb.openSum += v
|
||||||
|
tb.openCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tb *tierBuf) flush() {
|
||||||
|
if tb.openCount == 0 {
|
||||||
|
tb.open = false
|
||||||
|
return
|
||||||
}
|
}
|
||||||
h.metrics[metric] = buf
|
tb.samples = append(tb.samples, MetricSample{T: tb.openStart, V: tb.openSum / float64(tb.openCount)})
|
||||||
|
if len(tb.samples) > tb.capacity {
|
||||||
|
tb.samples = tb.samples[len(tb.samples)-tb.capacity:]
|
||||||
|
}
|
||||||
|
tb.open = false
|
||||||
|
tb.openStart = 0
|
||||||
|
tb.openSum = 0
|
||||||
|
tb.openCount = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
// drop removes the entire history for one metric. Used when a node is
|
// readSamples returns a copy of the closed buckets plus the still-open one, so
|
||||||
// deleted so its old samples don't linger forever in the singleton.
|
// the most recent point is visible before its bucket boundary closes.
|
||||||
func (h *metricHistory) drop(metric string) {
|
func (tb *tierBuf) readSamples() []MetricSample {
|
||||||
h.mu.Lock()
|
out := make([]MetricSample, len(tb.samples), len(tb.samples)+1)
|
||||||
delete(h.metrics, metric)
|
copy(out, tb.samples)
|
||||||
h.mu.Unlock()
|
if tb.openCount > 0 {
|
||||||
}
|
out = append(out, MetricSample{T: tb.openStart, V: tb.openSum / float64(tb.openCount)})
|
||||||
|
|
||||||
// snapshot returns a deep copy of every series, safe to serialize without
|
|
||||||
// holding the lock during disk I/O.
|
|
||||||
func (h *metricHistory) snapshot() map[string][]MetricSample {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
out := make(map[string][]MetricSample, len(h.metrics))
|
|
||||||
for k, v := range h.metrics {
|
|
||||||
cp := make([]MetricSample, len(v))
|
|
||||||
copy(cp, v)
|
|
||||||
out[k] = cp
|
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// restore replaces the in-memory series with a previously persisted set,
|
// series is the rollup ladder for one metric: a sample is fed to every tier.
|
||||||
// re-applying the per-series capacity cap so a tampered or oversized file
|
type series struct {
|
||||||
// can't grow the working set unbounded.
|
tiers []*tierBuf
|
||||||
func (h *metricHistory) restore(data map[string][]MetricSample) {
|
}
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
func newSeries() *series {
|
||||||
for k, v := range data {
|
s := &series{tiers: make([]*tierBuf, len(metricTiers))}
|
||||||
if len(v) > metricCapacityDefault {
|
for i, spec := range metricTiers {
|
||||||
v = v[len(v)-metricCapacityDefault:]
|
s.tiers[i] = &tierBuf{resolution: spec.resolution, capacity: spec.capacity}
|
||||||
}
|
}
|
||||||
h.metrics[k] = v
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *series) add(unixSec int64, v float64) {
|
||||||
|
for _, tb := range s.tiers {
|
||||||
|
tb.add(unixSec, v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// aggregate returns up to maxPoints buckets of size bucketSeconds,
|
// pickTier returns the finest tier whose window covers spanSeconds, falling back
|
||||||
// each bucket carrying the arithmetic mean of the underlying samples.
|
// to the coarsest (longest-window) tier when nothing covers it.
|
||||||
// Bucket alignment is to absolute Unix-second boundaries so two
|
func (s *series) pickTier(spanSeconds int64) *tierBuf {
|
||||||
// concurrent calls (e.g. two browser tabs) see identical x-axes.
|
for _, tb := range s.tiers {
|
||||||
func (h *metricHistory) aggregate(metric string, bucketSeconds int, maxPoints int) []map[string]any {
|
if int64(tb.resolution)*int64(tb.capacity) >= spanSeconds {
|
||||||
if bucketSeconds <= 0 || maxPoints <= 0 {
|
return tb
|
||||||
return []map[string]any{}
|
|
||||||
}
|
|
||||||
cutoff := time.Now().Add(-time.Duration(bucketSeconds*maxPoints) * time.Second).Unix()
|
|
||||||
|
|
||||||
h.mu.Lock()
|
|
||||||
hist := h.metrics[metric]
|
|
||||||
startIdx := 0
|
|
||||||
for i, h := range slices.Backward(hist) {
|
|
||||||
if h.T < cutoff {
|
|
||||||
startIdx = i + 1
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if startIdx >= len(hist) {
|
return s.tiers[len(s.tiers)-1]
|
||||||
h.mu.Unlock()
|
}
|
||||||
return []map[string]any{}
|
|
||||||
|
// metricHistory is a thread-safe, in-memory store of tiered series keyed by
|
||||||
|
// arbitrary strings. Three singletons live below: system-wide host metrics,
|
||||||
|
// per-node metrics, and xray expvar metrics.
|
||||||
|
type metricHistory struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
series map[string]*series
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMetricHistory() *metricHistory {
|
||||||
|
return &metricHistory{series: map[string]*series{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// append stores a single sample for the given metric across all tiers.
|
||||||
|
func (h *metricHistory) append(metric string, t time.Time, v float64) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
s := h.series[metric]
|
||||||
|
if s == nil {
|
||||||
|
s = newSeries()
|
||||||
|
h.series[metric] = s
|
||||||
}
|
}
|
||||||
tmp := make([]MetricSample, len(hist)-startIdx)
|
s.add(t.Unix(), v)
|
||||||
copy(tmp, hist[startIdx:])
|
}
|
||||||
|
|
||||||
|
// drop removes the entire history for one metric. Used when a node is deleted so
|
||||||
|
// its old samples don't linger forever in the singleton.
|
||||||
|
func (h *metricHistory) drop(metric string) {
|
||||||
|
h.mu.Lock()
|
||||||
|
delete(h.series, metric)
|
||||||
|
h.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregate returns up to maxPoints buckets of size bucketSeconds, each carrying
|
||||||
|
// the arithmetic mean of the underlying samples from the finest tier that covers
|
||||||
|
// the requested span. Bucket alignment is to absolute Unix-second boundaries so
|
||||||
|
// two concurrent calls see identical x-axes.
|
||||||
|
func (h *metricHistory) aggregate(metric string, bucketSeconds int, maxPoints int) []map[string]any {
|
||||||
|
empty := []map[string]any{}
|
||||||
|
if bucketSeconds <= 0 || maxPoints <= 0 {
|
||||||
|
return empty
|
||||||
|
}
|
||||||
|
span := int64(bucketSeconds) * int64(maxPoints)
|
||||||
|
cutoff := time.Now().Unix() - span
|
||||||
|
|
||||||
|
h.mu.Lock()
|
||||||
|
s := h.series[metric]
|
||||||
|
if s == nil {
|
||||||
|
h.mu.Unlock()
|
||||||
|
return empty
|
||||||
|
}
|
||||||
|
raw := s.pickTier(span).readSamples()
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
|
|
||||||
|
startIdx := len(raw)
|
||||||
|
for i := len(raw) - 1; i >= 0; i-- {
|
||||||
|
if raw[i].T < cutoff {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
startIdx = i
|
||||||
|
}
|
||||||
|
tmp := raw[startIdx:]
|
||||||
if len(tmp) == 0 {
|
if len(tmp) == 0 {
|
||||||
return []map[string]any{}
|
return empty
|
||||||
}
|
}
|
||||||
|
|
||||||
bSize := int64(bucketSeconds)
|
bSize := int64(bucketSeconds)
|
||||||
@@ -152,24 +209,79 @@ func (h *metricHistory) aggregate(metric string, bucketSeconds int, maxPoints in
|
|||||||
out = out[len(out)-maxPoints:]
|
out = out[len(out)-maxPoints:]
|
||||||
}
|
}
|
||||||
if out == nil {
|
if out == nil {
|
||||||
return []map[string]any{}
|
return empty
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// systemMetrics holds whole-host time series (cpu, mem, netUp, etc.)
|
// persistedTier and persistedSeries are the on-disk shape of a series. Tiers are
|
||||||
// fed by ServerService.RefreshStatus every 2s. nodeMetrics holds
|
// matched back by resolution on restore, so changing the ladder degrades
|
||||||
// per-node CPU/Mem fed by NodeHeartbeatJob every 10s. Both are
|
// gracefully (unmatched layers are dropped) instead of corrupting state.
|
||||||
// process-local — survival across panel restart is not required.
|
type persistedTier struct {
|
||||||
|
Resolution int
|
||||||
|
Samples []MetricSample
|
||||||
|
}
|
||||||
|
|
||||||
|
type persistedSeries struct {
|
||||||
|
Tiers []persistedTier
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshot returns a deep copy of every series' closed buckets, safe to
|
||||||
|
// serialize without holding the lock during disk I/O.
|
||||||
|
func (h *metricHistory) snapshot() map[string]persistedSeries {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
out := make(map[string]persistedSeries, len(h.series))
|
||||||
|
for k, s := range h.series {
|
||||||
|
ps := persistedSeries{Tiers: make([]persistedTier, len(s.tiers))}
|
||||||
|
for i, tb := range s.tiers {
|
||||||
|
cp := make([]MetricSample, len(tb.samples))
|
||||||
|
copy(cp, tb.samples)
|
||||||
|
ps.Tiers[i] = persistedTier{Resolution: tb.resolution, Samples: cp}
|
||||||
|
}
|
||||||
|
out[k] = ps
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// restore replaces the in-memory series with a previously persisted set,
|
||||||
|
// re-applying each tier's capacity cap so a tampered or oversized file can't grow
|
||||||
|
// the working set unbounded.
|
||||||
|
func (h *metricHistory) restore(data map[string]persistedSeries) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
for k, ps := range data {
|
||||||
|
s := newSeries()
|
||||||
|
for _, pt := range ps.Tiers {
|
||||||
|
for _, tb := range s.tiers {
|
||||||
|
if tb.resolution != pt.Resolution {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
samples := pt.Samples
|
||||||
|
if len(samples) > tb.capacity {
|
||||||
|
samples = samples[len(samples)-tb.capacity:]
|
||||||
|
}
|
||||||
|
tb.samples = samples
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.series[k] = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// systemMetrics holds whole-host time series (cpu, mem, netUp, etc.) fed by
|
||||||
|
// ServerService.RefreshStatus every 2s. nodeMetrics holds per-node CPU/Mem fed
|
||||||
|
// by NodeHeartbeatJob. xrayMetrics holds xray expvar series. Only systemMetrics
|
||||||
|
// is persisted; the others rebuild from live connections.
|
||||||
var (
|
var (
|
||||||
systemMetrics = newMetricHistory()
|
systemMetrics = newMetricHistory()
|
||||||
nodeMetrics = newMetricHistory()
|
nodeMetrics = newMetricHistory()
|
||||||
xrayMetrics = newMetricHistory()
|
xrayMetrics = newMetricHistory()
|
||||||
)
|
)
|
||||||
|
|
||||||
// SystemMetricKeys lists the metric names ServerService writes on every
|
// SystemMetricKeys lists the metric names ServerService writes on every status
|
||||||
// status sample. Exposed for documentation/test purposes; the
|
// sample. Exposed for documentation/test purposes; the controller validates
|
||||||
// controller validates incoming names against an allow-list.
|
// incoming names against an allow-list.
|
||||||
var SystemMetricKeys = []string{
|
var SystemMetricKeys = []string{
|
||||||
"cpu", "mem", "swap", "netUp", "netDown", "pktUp", "pktDown", "diskRead", "diskWrite", "diskUsage", "tcpCount", "udpCount", "online", "load1", "load5", "load15",
|
"cpu", "mem", "swap", "netUp", "netDown", "pktUp", "pktDown", "diskRead", "diskWrite", "diskUsage", "tcpCount", "udpCount", "online", "load1", "load5", "load15",
|
||||||
}
|
}
|
||||||
@@ -177,18 +289,13 @@ var SystemMetricKeys = []string{
|
|||||||
// NodeMetricKeys lists the per-node metric names NodeHeartbeatJob writes.
|
// NodeMetricKeys lists the per-node metric names NodeHeartbeatJob writes.
|
||||||
var NodeMetricKeys = []string{"cpu", "mem", "netUp", "netDown"}
|
var NodeMetricKeys = []string{"cpu", "mem", "netUp", "netDown"}
|
||||||
|
|
||||||
// XrayMetricKeys lists series sourced from xray's /debug/vars expvar
|
// XrayMetricKeys lists series sourced from xray's /debug/vars expvar endpoint.
|
||||||
// endpoint. Populated by XrayMetricsService.Sample on the same 2s cadence
|
|
||||||
// as the system metrics, but only when the xray config has a `metrics`
|
|
||||||
// block configured.
|
|
||||||
var XrayMetricKeys = []string{
|
var XrayMetricKeys = []string{
|
||||||
"xrAlloc", "xrSys", "xrHeapObjects", "xrNumGC", "xrPauseNs",
|
"xrAlloc", "xrSys", "xrHeapObjects", "xrNumGC", "xrPauseNs",
|
||||||
}
|
}
|
||||||
|
|
||||||
// systemMetricsStorePath is where the host time-series is persisted between
|
// systemMetricsStorePath is where the host time-series is persisted between
|
||||||
// restarts. It lives next to the database so a single volume mount carries
|
// restarts. It lives next to the database so a single volume mount carries both.
|
||||||
// both. Only systemMetrics is persisted — node and xray series are cheap to
|
|
||||||
// rebuild and tied to live connections.
|
|
||||||
func systemMetricsStorePath() string {
|
func systemMetricsStorePath() string {
|
||||||
return filepath.Join(config.GetDBFolderPath(), "system_metrics.gob")
|
return filepath.Join(config.GetDBFolderPath(), "system_metrics.gob")
|
||||||
}
|
}
|
||||||
@@ -216,8 +323,8 @@ func PersistSystemMetrics() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RestoreSystemMetrics loads a previously persisted host time-series on startup.
|
// RestoreSystemMetrics loads a previously persisted host time-series on startup.
|
||||||
// A missing file is not an error (first boot). Aggregation already windows by
|
// A missing file is not an error (first boot). A pre-tier flat snapshot is
|
||||||
// time, so any gap from downtime is handled by the readers.
|
// migrated by replaying its samples through the rollup.
|
||||||
func RestoreSystemMetrics() {
|
func RestoreSystemMetrics() {
|
||||||
path := systemMetricsStorePath()
|
path := systemMetricsStorePath()
|
||||||
f, err := os.Open(path)
|
f, err := os.Open(path)
|
||||||
@@ -227,11 +334,36 @@ func RestoreSystemMetrics() {
|
|||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer f.Close()
|
var data map[string]persistedSeries
|
||||||
var data map[string][]MetricSample
|
decErr := gob.NewDecoder(f).Decode(&data)
|
||||||
if err := gob.NewDecoder(f).Decode(&data); err != nil {
|
f.Close()
|
||||||
logger.Warning("decode system metrics failed:", err)
|
if decErr == nil {
|
||||||
|
systemMetrics.restore(data)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
systemMetrics.restore(data)
|
if migrateLegacySystemMetrics(path) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logger.Warning("decode system metrics failed:", decErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrateLegacySystemMetrics loads a pre-tier flat snapshot
|
||||||
|
// (map[string][]MetricSample) and replays it through append so the new tiers are
|
||||||
|
// seeded from the existing history instead of starting empty.
|
||||||
|
func migrateLegacySystemMetrics(path string) bool {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
var legacy map[string][]MetricSample
|
||||||
|
if err := gob.NewDecoder(f).Decode(&legacy); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for metric, samples := range legacy {
|
||||||
|
for _, p := range samples {
|
||||||
|
systemMetrics.append(metric, time.Unix(p.T, 0), p.V)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMetricMemoryFootprint(t *testing.T) {
|
||||||
|
const metrics = 16
|
||||||
|
|
||||||
|
retained := func(build func() any) uint64 {
|
||||||
|
runtime.GC()
|
||||||
|
runtime.GC()
|
||||||
|
var m0 runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&m0)
|
||||||
|
obj := build()
|
||||||
|
runtime.GC()
|
||||||
|
var m1 runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&m1)
|
||||||
|
runtime.KeepAlive(obj)
|
||||||
|
if m1.HeapAlloc < m0.HeapAlloc {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return m1.HeapAlloc - m0.HeapAlloc
|
||||||
|
}
|
||||||
|
|
||||||
|
fill := func(buf []MetricSample) {
|
||||||
|
for j := range buf {
|
||||||
|
buf[j] = MetricSample{T: int64(j), V: float64(j)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
oldFlat := retained(func() any {
|
||||||
|
m := make(map[string][]MetricSample, metrics)
|
||||||
|
for i := 0; i < metrics; i++ {
|
||||||
|
buf := make([]MetricSample, 86400)
|
||||||
|
fill(buf)
|
||||||
|
m[fmt.Sprintf("m%d", i)] = buf
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
})
|
||||||
|
|
||||||
|
newTiered := retained(func() any {
|
||||||
|
h := newMetricHistory()
|
||||||
|
for i := 0; i < metrics; i++ {
|
||||||
|
s := newSeries()
|
||||||
|
for _, tb := range s.tiers {
|
||||||
|
buf := make([]MetricSample, tb.capacity)
|
||||||
|
fill(buf)
|
||||||
|
tb.samples = buf
|
||||||
|
}
|
||||||
|
h.series[fmt.Sprintf("m%d", i)] = s
|
||||||
|
}
|
||||||
|
return h
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Logf("metric history footprint (16 system metrics, full):")
|
||||||
|
t.Logf(" before (flat 48h@2s): %d KiB", oldFlat/1024)
|
||||||
|
t.Logf(" after (tiered 7d): %d KiB", newTiered/1024)
|
||||||
|
if newTiered >= oldFlat {
|
||||||
|
t.Fatalf("expected tiered footprint smaller: old=%d new=%d", oldFlat, newTiered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTierBufRollupAveragesClosedBuckets(t *testing.T) {
|
||||||
|
tb := &tierBuf{resolution: 10, capacity: 100}
|
||||||
|
tb.add(0, 2)
|
||||||
|
tb.add(2, 4)
|
||||||
|
tb.add(5, 6)
|
||||||
|
tb.add(10, 10)
|
||||||
|
|
||||||
|
if len(tb.samples) != 1 || tb.samples[0].T != 0 || tb.samples[0].V != 4 {
|
||||||
|
t.Fatalf("expected one closed bucket {0,4}, got %+v", tb.samples)
|
||||||
|
}
|
||||||
|
|
||||||
|
got := tb.readSamples()
|
||||||
|
if len(got) != 2 || got[1].T != 10 || got[1].V != 10 {
|
||||||
|
t.Fatalf("expected open bucket {10,10} appended on read, got %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTierBufRespectsCapacity(t *testing.T) {
|
||||||
|
tb := &tierBuf{resolution: 1, capacity: 5}
|
||||||
|
for i := int64(0); i < 20; i++ {
|
||||||
|
tb.add(i, float64(i))
|
||||||
|
}
|
||||||
|
if len(tb.samples) != 5 {
|
||||||
|
t.Fatalf("expected closed buckets capped at 5, got %d", len(tb.samples))
|
||||||
|
}
|
||||||
|
if last := tb.samples[len(tb.samples)-1]; last.T != 18 {
|
||||||
|
t.Fatalf("expected last closed bucket T=18 (19 still open), got %d", last.T)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSeriesPickTierBySpan(t *testing.T) {
|
||||||
|
s := newSeries()
|
||||||
|
cases := []struct {
|
||||||
|
span int64
|
||||||
|
res int
|
||||||
|
}{
|
||||||
|
{120, 2},
|
||||||
|
{3600, 2},
|
||||||
|
{7200, 60},
|
||||||
|
{172800, 60},
|
||||||
|
{604800, 600},
|
||||||
|
{9999999, 600},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
if got := s.pickTier(c.span); got.resolution != c.res {
|
||||||
|
t.Errorf("span %d: expected resolution %d, got %d", c.span, c.res, got.resolution)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAggregateFineRealtime(t *testing.T) {
|
||||||
|
h := newMetricHistory()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for i := int64(59); i >= 0; i-- {
|
||||||
|
h.append("cpu", time.Unix(now-i*2, 0), float64(100-i))
|
||||||
|
}
|
||||||
|
|
||||||
|
out := h.aggregate("cpu", 2, 60)
|
||||||
|
if len(out) == 0 {
|
||||||
|
t.Fatalf("expected non-empty realtime aggregate")
|
||||||
|
}
|
||||||
|
if _, ok := out[len(out)-1]["v"].(float64); !ok {
|
||||||
|
t.Fatalf("expected float64 value, got %T", out[len(out)-1]["v"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAggregateLongSpanUsesCoarseTier(t *testing.T) {
|
||||||
|
h := newMetricHistory()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for i := int64(0); i < 200; i++ {
|
||||||
|
ts := now - (200-i)*600
|
||||||
|
h.append("cpu", time.Unix(ts, 0), float64(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
out := h.aggregate("cpu", 10080, 60)
|
||||||
|
if len(out) == 0 {
|
||||||
|
t.Fatalf("expected non-empty 7d aggregate from the archive tier")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSnapshotRestoreRoundTrip(t *testing.T) {
|
||||||
|
h := newMetricHistory()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for i := int64(0); i < 10; i++ {
|
||||||
|
h.append("cpu", time.Unix(now-(9-i)*2, 0), float64(i))
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 := newMetricHistory()
|
||||||
|
h2.restore(h.snapshot())
|
||||||
|
|
||||||
|
if out := h2.aggregate("cpu", 2, 60); len(out) == 0 {
|
||||||
|
t.Fatalf("expected restored series to aggregate")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAggregateMissingMetricIsEmpty(t *testing.T) {
|
||||||
|
h := newMetricHistory()
|
||||||
|
if out := h.aggregate("nope", 2, 60); len(out) != 0 {
|
||||||
|
t.Fatalf("expected empty result for unknown metric, got %d points", len(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -166,15 +166,15 @@ const xrayVersionsCacheTTL = 15 * time.Minute
|
|||||||
// callers from triggering arbitrary aggregation work and keeps the
|
// callers from triggering arbitrary aggregation work and keeps the
|
||||||
// frontend's bucket selector self-documenting.
|
// frontend's bucket selector self-documenting.
|
||||||
var allowedHistoryBuckets = map[int]bool{
|
var allowedHistoryBuckets = map[int]bool{
|
||||||
2: true, // Real-time view
|
2: true, // 2m
|
||||||
30: true, // 30s intervals
|
30: true, // 30m
|
||||||
60: true, // 1m intervals
|
60: true, // 1h
|
||||||
120: true, // 2m intervals
|
180: true, // 3h
|
||||||
180: true, // 3m intervals
|
360: true, // 6h
|
||||||
300: true, // 5m intervals
|
720: true, // 12h
|
||||||
720: true, // 12m intervals
|
1440: true, // 24h
|
||||||
1440: true, // 24m intervals
|
2880: true, // 2d
|
||||||
2880: true, // 48m intervals
|
10080: true, // 7d
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsAllowedHistoryBucket reports whether a bucket-seconds value is in the
|
// IsAllowedHistoryBucket reports whether a bucket-seconds value is in the
|
||||||
|
|||||||
@@ -398,6 +398,10 @@ func (s *Server) startTask(restartXray bool) {
|
|||||||
|
|
||||||
if mins := sys.MemoryReleaseIntervalMinutes(); mins > 0 {
|
if mins := sys.MemoryReleaseIntervalMinutes(); mins > 0 {
|
||||||
s.cron.AddJob(fmt.Sprintf("@every %dm", mins), job.NewMemoryReleaseJob())
|
s.cron.AddJob(fmt.Sprintf("@every %dm", mins), job.NewMemoryReleaseJob())
|
||||||
|
go func() {
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
job.NewMemoryReleaseJob().Run()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user