feat: add iowait, TCP connection states, and file descriptor metrics

- cpu.iowait_percent: % of time CPU is waiting on disk I/O
- tcp: established/time_wait/close_wait/listen counts; max_time_wait
  system limit included on Linux for context
- file_descriptors: used vs max (Linux reads /proc/sys/fs/file-nr,
  macOS reads kern.openfiles / kern.maxfiles via sysctl)

These three metrics directly surface the most common causes of
unexplained PHP/Laravel timeouts at the system level.
This commit is contained in:
Joel Male
2026-02-25 15:35:43 +10:00
parent 1a0c3e072c
commit bd63686312
2 changed files with 112 additions and 1 deletions
+100 -1
View File
@@ -13,6 +13,7 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strconv"
"strings" "strings"
"syscall" "syscall"
"time" "time"
@@ -81,7 +82,9 @@ type Metrics struct {
Disks []DiskMetrics `json:"disks"` Disks []DiskMetrics `json:"disks"`
DiskIO DiskIOMetrics `json:"disk_io"` DiskIO DiskIOMetrics `json:"disk_io"`
Network *NetworkMetrics `json:"network"` Network *NetworkMetrics `json:"network"`
Processes []ProcessInfo `json:"processes"` Processes []ProcessInfo `json:"processes"`
TCP *TCPStats `json:"tcp,omitempty"`
FileDescriptors *FileDescriptors `json:"file_descriptors,omitempty"`
} }
type SystemInfo struct { type SystemInfo struct {
@@ -102,10 +105,24 @@ type LoadMetrics struct {
type CPUMetrics struct { type CPUMetrics struct {
Percent float64 `json:"percent"` Percent float64 `json:"percent"`
Iowait float64 `json:"iowait_percent"`
PerCore []float64 `json:"per_core"` PerCore []float64 `json:"per_core"`
Count int `json:"core_count"` Count int `json:"core_count"`
} }
type TCPStats struct {
Established int `json:"established"`
TimeWait int `json:"time_wait"`
CloseWait int `json:"close_wait"`
Listen int `json:"listen"`
MaxTimeWait int `json:"max_time_wait,omitempty"` // Linux only
}
type FileDescriptors struct {
Used int `json:"used"`
Max int `json:"max"`
}
type MemoryMetrics struct { type MemoryMetrics struct {
Total uint64 `json:"total"` Total uint64 `json:"total"`
Available uint64 `json:"available"` Available uint64 `json:"available"`
@@ -877,6 +894,15 @@ func collectMetrics(config *Config) (*Metrics, error) {
metrics.CPU.Percent = sum / float64(len(cpuPerCore)) metrics.CPU.Percent = sum / float64(len(cpuPerCore))
} }
cpuTimes, err := cpu.Times(false)
if err == nil && len(cpuTimes) > 0 {
t := cpuTimes[0]
total := t.User + t.System + t.Idle + t.Iowait + t.Irq + t.Softirq + t.Steal + t.Nice + t.Guest + t.GuestNice
if total > 0 {
metrics.CPU.Iowait = (t.Iowait / total) * 100
}
}
cpuCount, err := cpu.Counts(true) cpuCount, err := cpu.Counts(true)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get CPU count: %w", err) return nil, fmt.Errorf("failed to get CPU count: %w", err)
@@ -1005,9 +1031,82 @@ func collectMetrics(config *Config) (*Metrics, error) {
metrics.Processes = processes metrics.Processes = processes
} }
metrics.TCP = collectTCPStats()
metrics.FileDescriptors = collectFileDescriptors()
return metrics, nil return metrics, nil
} }
func collectTCPStats() *TCPStats {
conns, err := net.Connections("tcp")
if err != nil {
return nil
}
conns6, _ := net.Connections("tcp6")
conns = append(conns, conns6...)
stats := &TCPStats{}
for _, c := range conns {
switch c.Status {
case "ESTABLISHED":
stats.Established++
case "TIME_WAIT":
stats.TimeWait++
case "CLOSE_WAIT":
stats.CloseWait++
case "LISTEN":
stats.Listen++
}
}
// Linux: include the system TIME_WAIT bucket limit for context
if runtime.GOOS == "linux" {
if data, err := os.ReadFile("/proc/sys/net/ipv4/tcp_max_tw_buckets"); err == nil {
if v, err := strconv.Atoi(strings.TrimSpace(string(data))); err == nil {
stats.MaxTimeWait = v
}
}
}
return stats
}
func collectFileDescriptors() *FileDescriptors {
switch runtime.GOOS {
case "linux":
// /proc/sys/fs/file-nr: allocated unused max
data, err := os.ReadFile("/proc/sys/fs/file-nr")
if err != nil {
return nil
}
fields := strings.Fields(string(data))
if len(fields) < 3 {
return nil
}
used, err1 := strconv.Atoi(fields[0])
max, err2 := strconv.Atoi(fields[2])
if err1 != nil || err2 != nil {
return nil
}
return &FileDescriptors{Used: used, Max: max}
case "darwin":
openOut, err1 := exec.Command("sysctl", "-n", "kern.openfiles").Output()
maxOut, err2 := exec.Command("sysctl", "-n", "kern.maxfiles").Output()
if err1 != nil || err2 != nil {
return nil
}
used, err1 := strconv.Atoi(strings.TrimSpace(string(openOut)))
max, err2 := strconv.Atoi(strings.TrimSpace(string(maxOut)))
if err1 != nil || err2 != nil {
return nil
}
return &FileDescriptors{Used: used, Max: max}
}
return nil
}
func collectTopProcesses() ([]ProcessInfo, error) { func collectTopProcesses() ([]ProcessInfo, error) {
pids, err := process.Pids() pids, err := process.Pids()
if err != nil { if err != nil {
+12
View File
@@ -21,6 +21,7 @@
}, },
"cpu": { "cpu": {
"percent": 34.5, "percent": 34.5,
"iowait_percent": 4.2,
"per_core": [ "per_core": [
42.1, 42.1,
28.3, 28.3,
@@ -74,6 +75,17 @@
"packets_sent_per_sec": 95.3, "packets_sent_per_sec": 95.3,
"packets_recv_per_sec": 140.7 "packets_recv_per_sec": 140.7
}, },
"tcp": {
"established": 42,
"time_wait": 18,
"close_wait": 3,
"listen": 8,
"max_time_wait": 32768
},
"file_descriptors": {
"used": 1248,
"max": 98304
},
"processes": [ "processes": [
{ {
"pid": 1234, "pid": 1234,