//! System metrics for the System Monitor module (port of `readSystemMetrics`, //! `parseSystemMetricsOutput`, `parseLinuxMemoryDetail` and `sessions/windowsMetrics.js`). //! The Linux script is never sent to a Windows shell: the platform is detected first. use crate::core::error::{CoreError, CoreResult}; use crate::core::ssh::platform::detect_platform; use crate::core::ssh::session::{SessionManager, SessionRecord}; use crate::core::ssh::windows_exec::run_windows_script; use serde_json::{json, Map, Value}; use std::sync::Arc; use std::time::Duration; pub const METRICS_COMMAND: &str = include_str!("scripts/linux_metrics.sh"); pub const WINDOWS_METRICS_SCRIPT: &str = include_str!("scripts/windows_metrics.ps1"); const LIST_PREFIXES: [&str; 9] = ["networkInterface=", "meminfo=", "swapArea=", "vmstat=", "zram=", "zswap=", "cpuPackage=", "gpu=", "nvidiaGpu="]; /// JS `Number(value)` for metric strings ("" → 0, invalid → NaN). fn js_number(value: &str) -> f64 { let trimmed = value.trim(); if trimmed.is_empty() { return 0.0; } trimmed.parse::().unwrap_or(f64::NAN) } pub fn metric_number(value: &str, min: f64, max: f64) -> f64 { let n = js_number(value); if n.is_finite() { n.max(min).min(max) } else { min } } fn metric_value(n: f64) -> Value { if n.fract() == 0.0 && n.abs() < 9.0e15 { json!(n as i64) } else { json!(n) } } pub fn metric_text(value: &str, fallback: &str, max: usize) -> String { let replaced: String = value.chars().map(|c| if (c as u32) < 0x20 || c as u32 == 0x7f || c == '|' { ' ' } else { c }).collect(); let normalized: String = replaced.split_whitespace().collect::>().join(" ").chars().take(max).collect(); if normalized.is_empty() { fallback.to_string() } else { normalized } } fn parse_core_types(value: &str) -> Vec { value .split(',') .take(16) .filter_map(|entry| { let index = entry.rfind(':')?; if index == 0 { return None; } let name = metric_text(&entry[..index], "", 80); let count = metric_number(&entry[index + 1..], 0.0, 4096.0).floor(); (!name.is_empty() && count > 0.0).then(|| json!({ "name": name, "count": count as i64 })) }) .collect() } /// A sensor value or null; `[N/A]` and `[Not Supported]` from nvidia-smi are not measurements. fn sensor(value: &str, scale: f64, max: f64) -> Value { match value.trim().parse::() { Ok(n) if n.is_finite() && n >= 0.0 => { let scaled = n * scale; metric_value((scaled.min(max) * 10.0).round() / 10.0) } _ => Value::Null, } } fn parse_cpu_packages(lines: &[&str]) -> Vec { let mut packages: Vec = lines .iter() .filter(|l| l.starts_with("cpuPackage=")) .take(64) .map(|line| { let f: Vec<&str> = line["cpuPackage=".len()..].split('|').collect(); let get = |i: usize| f.get(i).copied().unwrap_or(""); json!({ "id": metric_text(get(0), "0", 16), "model": metric_text(get(1), "", 240), "logicalCores": metric_number(get(2), 0.0, 4096.0).floor() as i64, "physicalCores": metric_number(get(3), 0.0, 4096.0).floor() as i64, "usagePercent": sensor(get(4), 1.0, 100.0), "frequencyMhz": sensor(get(5), 1.0, 1.0e6), "temperatureC": sensor(get(6), 0.001, 400.0) }) }) .collect(); packages.sort_by_key(|p| p["id"].as_str().and_then(|id| id.parse::().ok()).unwrap_or(i64::MAX)); packages } fn empty_gpu(id: &str) -> Value { json!({ "id": id, "name": "", "vendorId": "", "deviceId": "", "driver": "", "driverVersion": "", "source": "none", "usagePercent": null, "memoryUsagePercent": null, "memoryTotal": null, "memoryUsed": null, "temperatureC": null, "powerW": null, "powerLimitW": null, "clockMhz": null, "fanPercent": null }) } /// One `nvidia-smi --query-gpu` CSV row (the same query on Linux and Windows). fn apply_nvidia_row(gpu: &mut Value, fields: &[&str]) { let get = |i: usize| fields.get(i).copied().unwrap_or(""); gpu["name"] = json!(metric_text(get(1), "NVIDIA GPU", 160)); gpu["usagePercent"] = sensor(get(2), 1.0, 100.0); gpu["memoryUsagePercent"] = sensor(get(3), 1.0, 100.0); gpu["memoryTotal"] = sensor(get(4), 1_048_576.0, f64::MAX); gpu["memoryUsed"] = sensor(get(5), 1_048_576.0, f64::MAX); gpu["temperatureC"] = sensor(get(6), 1.0, 400.0); gpu["powerW"] = sensor(get(7), 1.0, 1.0e6); gpu["powerLimitW"] = sensor(get(8), 1.0, 1.0e6); gpu["clockMhz"] = sensor(get(9), 1.0, 1.0e6); gpu["fanPercent"] = sensor(get(10), 1.0, 100.0); gpu["driverVersion"] = json!(metric_text(get(11), "", 40)); gpu["driver"] = json!("nvidia"); gpu["source"] = json!("nvidia-smi"); } /// `0000:01:00.0` from nvidia-smi's `00000000:01:00.0`, to match the sysfs device name. fn pci_id(bus_id: &str) -> String { let lower = bus_id.trim().to_lowercase(); lower.get(lower.len().saturating_sub(12)..).unwrap_or(&lower).to_string() } /// Virtual and BMC adapters that lspci's database often lacks (or lspci is not installed at all). fn known_gpu_name(vendor: &str, device: &str) -> Option<&'static str> { Some(match (vendor, device) { ("1234", "1111") => "QEMU Standard VGA", ("1b36", "0100") => "QEMU QXL VGA", ("1af4", "1050") => "Red Hat Virtio 1.0 GPU", ("1013", "00b8") => "Cirrus Logic GD 5446 (QEMU)", ("15ad", "0405") => "VMware SVGA II", ("15ad", "0406") => "VMware SVGA 3D", ("80ee", "beef") => "VirtualBox Graphics Adapter", ("1414", "5353") => "Microsoft Hyper-V Virtual VGA", ("1a03", "2000") => "ASPEED Graphics", _ => return None, }) } fn gpu_vendor_name(vendor: &str) -> Option<&'static str> { Some(match vendor { "10de" => "NVIDIA", "1002" => "AMD", "8086" => "Intel", "1a03" => "ASPEED", "102b" => "Matrox", "1af4" | "1b36" => "Red Hat", "1234" => "QEMU", "15ad" => "VMware", "80ee" => "VirtualBox", "1013" => "Cirrus Logic", "1414" => "Microsoft", _ => return None, }) } /// lspci prints `Vendor 1234 Device 1111` for IDs missing from its database. fn gpu_display_name(reported: &str, vendor: &str, device: &str) -> String { let unnamed = reported.is_empty() || (reported.starts_with("Vendor ") && reported.contains(" Device ")); if !unnamed { return reported.to_string(); } match (known_gpu_name(vendor, device), gpu_vendor_name(vendor)) { (Some(name), _) => name.to_string(), (None, Some(name)) => format!("{name} GPU"), _ => reported.to_string(), } } fn parse_gpus(lines: &[&str]) -> Vec { let mut gpus: Vec = lines .iter() .filter(|l| l.starts_with("gpu=")) .take(32) .map(|line| { let f: Vec<&str> = line["gpu=".len()..].split('|').collect(); let get = |i: usize| f.get(i).copied().unwrap_or(""); let mut gpu = empty_gpu(&metric_text(get(0), "", 32)); gpu["vendorId"] = json!(metric_text(get(1), "", 16).trim_start_matches("0x")); gpu["deviceId"] = json!(metric_text(get(2), "", 16).trim_start_matches("0x")); gpu["driver"] = json!(metric_text(get(3), "", 40)); let (vendor, device) = (metric_text(get(1), "", 16).trim_start_matches("0x").to_lowercase(), metric_text(get(2), "", 16).trim_start_matches("0x").to_lowercase()); gpu["name"] = json!(gpu_display_name(&metric_text(get(4), "", 160), &vendor, &device)); gpu["usagePercent"] = sensor(get(5), 1.0, 100.0); gpu["memoryUsagePercent"] = sensor(get(6), 1.0, 100.0); gpu["memoryTotal"] = sensor(get(7), 1.0, f64::MAX); gpu["memoryUsed"] = sensor(get(8), 1.0, f64::MAX); gpu["temperatureC"] = sensor(get(9), 0.001, 400.0); gpu["powerW"] = sensor(get(10), 0.000_001, 1.0e6); gpu["powerLimitW"] = sensor(get(11), 0.000_001, 1.0e6); gpu["clockMhz"] = sensor(get(12), 0.000_001, 1.0e6); gpu["fanPercent"] = sensor(get(13), 100.0 / 255.0, 100.0); if ["usagePercent", "memoryTotal", "temperatureC", "powerW", "clockMhz"].iter().any(|k| !gpu[*k].is_null()) { gpu["source"] = json!("sysfs"); } gpu }) .collect(); for line in lines.iter().filter(|l| l.starts_with("nvidiaGpu=")).take(32) { let fields: Vec<&str> = line["nvidiaGpu=".len()..].split('|').collect(); let id = pci_id(fields.first().copied().unwrap_or("")); match gpus.iter_mut().find(|gpu| gpu["id"] == id.as_str()) { Some(gpu) => apply_nvidia_row(gpu, &fields), None => { let mut gpu = empty_gpu(&id); apply_nvidia_row(&mut gpu, &fields); gpus.push(gpu); } } } gpus } fn parse_interfaces(lines: &[&str], default_interface: &str) -> Vec { lines .iter() .filter(|l| l.starts_with("networkInterface=")) .take(64) .map(|line| { let fields: Vec<&str> = line["networkInterface=".len()..].split('|').collect(); let get = |i: usize| fields.get(i).copied().unwrap_or(""); let name = metric_text(get(0), "unknown", 64); let addresses: Vec = get(5).split(',').take(16).map(|a| metric_text(a, "", 128)).filter(|a| !a.is_empty()).collect(); json!({ "name": name, "state": metric_text(get(1), "unknown", 32).to_lowercase(), "mac": metric_text(get(2), "", 64), "rxBytes": metric_value(metric_number(get(3), 0.0, f64::MAX)), "txBytes": metric_value(metric_number(get(4), 0.0, f64::MAX)), "addresses": addresses, "isDefault": name == default_interface }) }) .collect() } pub fn parse_linux_memory_detail(lines: &[&str]) -> Value { let mut meminfo: Map = Map::new(); for line in lines.iter().filter(|l| l.starts_with("meminfo=")) { let mut parts = line[8..].splitn(2, '|'); let (key, value) = (parts.next().unwrap_or(""), parts.next().unwrap_or("")); let n = js_number(value); if !key.is_empty() && key.chars().all(|c| c.is_ascii_alphabetic() || c == '_' || c == '(' || c == ')') && n.is_finite() { meminfo.insert(key.into(), metric_value(n)); } } let total = meminfo.get("MemTotal").and_then(Value::as_f64).unwrap_or(0.0); if total == 0.0 { return Value::Null; } let get = |k: &str| meminfo.get(k).cloned().unwrap_or(Value::Null); let num = |k: &str| meminfo.get(k).and_then(Value::as_f64).unwrap_or(0.0); let used = (total - num("MemFree") - num("Buffers") - num("Cached") - num("SReclaimable")).max(0.0); let mut vmstat: Map = Map::new(); for line in lines.iter().filter(|l| l.starts_with("vmstat=")) { let mut parts = line[7..].splitn(2, '|'); let key = parts.next().unwrap_or("").to_string(); let n = js_number(parts.next().unwrap_or("")); if n.is_finite() { vmstat.insert(key, metric_value(n)); } } let swap_areas: Vec = lines .iter() .filter(|l| l.starts_with("swapArea=")) .take(32) .map(|line| { let f: Vec<&str> = line[9..].split('|').collect(); let get = |i: usize| f.get(i).copied().unwrap_or(""); let path = get(0).replace("\\040", " "); let is_zram = path.strip_prefix("/dev/zram").map(|n| !n.is_empty() && n.chars().all(|c| c.is_ascii_digit())).unwrap_or(false); let priority = js_number(get(4)); json!({ "name": metric_text(&path, "", 255), "type": if is_zram { "zram".to_string() } else { metric_text(get(1), "unknown", 16) }, "size": metric_value(metric_number(get(2), 0.0, f64::MAX)), "used": metric_value(metric_number(get(3), 0.0, f64::MAX)), "priority": if priority.is_finite() { metric_value(priority) } else { Value::Null } }) }) .collect(); let zram: Vec = lines .iter() .filter(|l| l.starts_with("zram=")) .take(16) .map(|line| { let f: Vec<&str> = line[5..].split('|').collect(); let get = |i: usize| f.get(i).copied().unwrap_or(""); let stat: Vec = get(2).split(',').filter(|s| !s.is_empty()).map(js_number).collect(); let at = |i: usize| stat.get(i).map(|v| metric_value(*v)).unwrap_or(Value::Null); let algorithm = metric_text(get(3), "", 32); json!({ "device": metric_text(get(0), "", 32), "diskSize": metric_value(metric_number(get(1), 0.0, f64::MAX)), "originalData": at(0), "compressedData": at(1), "memoryUsed": at(2), "memoryLimit": stat.get(3).filter(|v| **v != 0.0 && v.is_finite()).map(|v| metric_value(*v)).unwrap_or(Value::Null), "algorithm": if algorithm.is_empty() { Value::Null } else { json!(algorithm) } }) }) .collect(); let zswap = lines.iter().find(|l| l.starts_with("zswap=")).map(|line| { let f: Vec<&str> = line[6..].split('|').collect(); let get = |i: usize| f.get(i).copied().unwrap_or(""); let compressor = metric_text(get(1), "", 32); let pool = js_number(get(2)); json!({ "enabled": get(0).starts_with(['Y', 'y', '1']), "compressor": if compressor.is_empty() { Value::Null } else { json!(compressor) }, "maxPoolPercent": if pool.is_finite() && pool != 0.0 { metric_value(pool) } else { Value::Null } }) }); let paging = |k: &str| vmstat.get(k).cloned().unwrap_or(Value::Null); json!({ "source": "/proc/meminfo", "total": metric_value(total), "free": metric_value(num("MemFree")), "available": get("MemAvailable"), "used": metric_value(used), "buffers": metric_value(num("Buffers")), "cached": metric_value(num("Cached")), "shmem": get("Shmem"), "swapCached": get("SwapCached"), "slab": get("Slab"), "slabReclaimable": get("SReclaimable"), "slabUnreclaimable": get("SUnreclaim"), "kernelStack": get("KernelStack"), "pageTables": get("PageTables"), "vmallocUsed": get("VmallocUsed"), "dirty": get("Dirty"), "writeback": get("Writeback"), "commit": { "committed": get("Committed_AS"), "limit": get("CommitLimit") }, "swapTotal": get("SwapTotal"), "swapFree": get("SwapFree"), "zswapPool": get("Zswap"), "zswapStored": get("Zswapped"), "paging": { "swapInPages": paging("pswpin"), "swapOutPages": paging("pswpout"), "majorFaults": paging("pgmajfault") }, "swapAreas": swap_areas, "zram": zram, "zswap": zswap.unwrap_or(Value::Null) }) } pub fn parse_system_metrics_output(stdout: &str, fallback_hostname: &str) -> Value { let lines: Vec<&str> = stdout.split('\n').map(|l| l.trim_end_matches('\r')).filter(|l| !l.is_empty()).collect(); let mut values: Map = Map::new(); for line in lines.iter().filter(|l| !LIST_PREFIXES.iter().any(|p| l.starts_with(p))) { match line.find('=') { Some(index) if index > 0 => values.insert(line[..index].to_string(), json!(line[index + 1..])), _ => values.insert(line.to_string(), json!("")), }; } let v = |k: &str| values.get(k).and_then(Value::as_str).unwrap_or("").to_string(); let disk: Vec = { let d = v("disk"); (if d.is_empty() { "0:0:0".to_string() } else { d }).split(':').map(|x| metric_number(x, 0.0, f64::MAX)).collect() }; let disk_at = |i: usize| disk.get(i).copied().unwrap_or(0.0); let memory_total = metric_number(&v("memoryTotal"), 0.0, f64::MAX); let memory_available = metric_number(&v("memoryAvailable"), 0.0, memory_total); let swap_total = metric_number(&v("swapTotal"), 0.0, f64::MAX); let swap_free = metric_number(&v("swapFree"), 0.0, swap_total); let default_interface = metric_text(&v("networkDefault"), "", 64); let detail = parse_linux_memory_detail(&lines); let page_size = metric_number(&v("pageSize"), 0.0, (1u64 << 30) as f64); json!({ "os": "linux", "loadAverageSupported": true, "pageSize": if page_size == 0.0 { Value::Null } else { metric_value(page_size) }, "memoryDetail": detail, "swapAreas": detail.get("swapAreas").cloned().unwrap_or_else(|| json!([])), "platform": metric_text(&v("platform"), "unknown", 240), "kernel": metric_text(&v("kernel"), "unknown", 240), "hostname": metric_text(&v("hostname"), fallback_hostname, 240), "uptimeSeconds": metric_value(metric_number(&v("uptimeSeconds"), 0.0, f64::MAX)), "load1": metric_value(metric_number(&v("load1"), 0.0, f64::MAX)), "memory": { "total": metric_value(memory_total), "available": metric_value(memory_available) }, "swap": { "total": metric_value(swap_total), "used": metric_value((swap_total - swap_free).max(0.0)), "free": metric_value(swap_free) }, "disk": { "mount": "/", "total": metric_value(disk_at(0)), "used": metric_value(disk_at(0).min(disk_at(1))), "available": metric_value(disk_at(0).min(disk_at(2))) }, "cpu": { "usagePercent": metric_value(metric_number(&v("cpuUsage"), 0.0, 100.0)), "architecture": metric_text(&v("cpuArchitecture"), "unknown", 240), "model": metric_text(&v("cpuModel"), "unknown", 240), "logicalCores": metric_number(&v("cpuLogical"), 0.0, 4096.0).floor() as i64, "physicalCores": metric_number(&v("cpuPhysical"), 0.0, 4096.0).floor() as i64, "sockets": metric_number(&v("cpuSockets"), 0.0, 256.0).floor() as i64, "coreTypes": parse_core_types(&v("cpuCoreTypes")), "packages": parse_cpu_packages(&lines) }, "gpus": parse_gpus(&lines), "network": { "defaultInterface": default_interface, "interfaces": parse_interfaces(&lines, &default_interface) } }) } // --------------------------------------------------------------------------------------------- // Windows // --------------------------------------------------------------------------------------------- fn finite(value: Option<&Value>) -> Option { match value? { Value::Number(n) => n.as_f64().filter(|v| v.is_finite()), Value::String(s) if !s.is_empty() => s.trim().parse::().ok().filter(|v| v.is_finite()), _ => None, } } fn text(value: Option<&Value>, limit: usize) -> String { let raw = match value { Some(Value::String(s)) => s.clone(), Some(Value::Null) | None => String::new(), Some(other) => other.to_string(), }; raw.chars().map(|c| if (c as u32) < 0x20 || c as u32 == 0x7f { ' ' } else { c }).collect::().trim().chars().take(limit).collect() } fn big(value: Option<&Value>) -> Option { match value? { Value::String(s) => s.trim().parse::().ok(), Value::Number(n) => n.as_i64().map(i128::from), _ => None, } } pub fn cpu_usage(previous: Option<&Value>, current: Option<&Value>) -> Option { let (p, c) = (previous?, current?); let busy = (big(c.get("busyInverse"))? - big(p.get("busyInverse"))?) as f64; let time = (big(c.get("time"))? - big(p.get("time"))?) as f64; if time <= 0.0 { return None; } Some(((1.0 - busy / time) * 100.0).clamp(0.0, 100.0)) } fn rate(previous: Option<&Value>, current: &Value, key: &str) -> Value { let Some(previous) = previous else { return Value::Null }; let Some(frequency) = finite(current.get("frequency")).filter(|f| *f != 0.0) else { return Value::Null }; let (Some(count_now), Some(count_before), Some(t_now), Some(t_before)) = (big(current.get(key)), big(previous.get(key)), big(current.get("time")), big(previous.get("time"))) else { return Value::Null; }; let count = (count_now - count_before) as f64; let seconds = (t_now - t_before) as f64 / frequency; if seconds > 0.0 && count >= 0.0 { json!(count / seconds) } else { Value::Null } } fn opt(v: Option) -> Value { v.map(metric_value).unwrap_or(Value::Null) } fn windows_cpu_packages(raw: &Value) -> Vec { raw.get("cpuPackages").and_then(Value::as_array).map(|list| { list.iter().take(64).map(|p| json!({ "id": text(p.get("id"), 16).trim_start_matches("CPU"), "model": text(p.get("name"), 240), "logicalCores": opt(finite(p.get("logical"))), "physicalCores": opt(finite(p.get("cores"))), "usagePercent": opt(finite(p.get("load")).map(|v| v.clamp(0.0, 100.0))), "frequencyMhz": opt(finite(p.get("mhz")).filter(|v| *v > 0.0)), "temperatureC": null })).collect() }).unwrap_or_default() } /// Win32_VideoController lists the adapters; nvidia-smi rows join by name, in order. The Windows GPU /// counters are keyed by an adapter LUID that WMI does not link to a controller, so they count only /// when one real adapter and one LUID leave no doubt. Otherwise the value stays unknown. fn windows_gpus(raw: &Value, static_info: &Value) -> Vec { let controllers = static_info.get("gpus").and_then(Value::as_array).cloned().unwrap_or_default(); let mut rows: Vec> = raw.get("nvidia").and_then(Value::as_array).map(|list| { list.iter().filter_map(Value::as_str).map(|row| row.split(',').map(|f| f.trim().to_string()).collect()).collect() }).unwrap_or_default(); let mut gpus: Vec = controllers.iter().take(32).enumerate().map(|(index, c)| { let name = text(c.get("name"), 160); let pnp = text(c.get("pnp"), 200); let id_part = |tag: &str| pnp.split(tag).nth(1).map(|rest| rest.chars().take(4).collect::().to_lowercase()).unwrap_or_default(); let mut gpu = empty_gpu(&format!("gpu{index}")); gpu["name"] = json!(name); gpu["vendorId"] = json!(id_part("VEN_")); gpu["deviceId"] = json!(id_part("DEV_")); gpu["driverVersion"] = json!(text(c.get("driverVersion"), 40)); // AdapterRAM is 32-bit: a value at the ceiling only means "4 GiB or more". gpu["memoryTotal"] = opt(finite(c.get("adapterRam")).filter(|v| *v > 0.0 && *v < 4_292_870_144.0)); if let Some(position) = rows.iter().position(|row| row.get(1).is_some_and(|n| n.eq_ignore_ascii_case(&name))) { let row = rows.remove(position); let fields: Vec<&str> = row.iter().map(String::as_str).collect(); apply_nvidia_row(&mut gpu, &fields); } gpu }).collect(); for row in &rows { let fields: Vec<&str> = row.iter().map(String::as_str).collect(); let mut gpu = empty_gpu(&pci_id(fields.first().copied().unwrap_or(""))); apply_nvidia_row(&mut gpu, &fields); gpus.push(gpu); } let real = |gpu: &Value| !gpu["vendorId"].as_str().unwrap_or("").is_empty() && gpu["vendorId"] != "1414"; let counters = raw.get("gpuCounters").and_then(Value::as_array).cloned().unwrap_or_default(); if counters.len() == 1 && gpus.iter().filter(|gpu| real(gpu)).count() == 1 { if let Some(gpu) = gpus.iter_mut().find(|gpu| real(gpu) && gpu["source"] == "none") { let c = &counters[0]; gpu["usagePercent"] = opt(finite(c.get("usage")).map(|v| v.clamp(0.0, 100.0))); gpu["memoryUsed"] = opt(finite(c.get("dedicated"))); gpu["source"] = json!("windows-counters"); } } gpus } pub fn normalize_windows_metrics(raw: &Value, static_info: &Value, state: &Value, fallback_hostname: &str) -> Value { let s = static_info; let total = finite(raw.get("memoryTotal")).or_else(|| finite(s.get("memoryTotal"))); let available = finite(raw.get("memoryAvailable")); let counters = raw.get("memoryCounters").filter(|c| c.is_object()); let usage = cpu_usage(state.get("cpuRaw"), raw.get("cpuRaw")); let pagefiles: Option> = raw.get("pagefiles").and_then(Value::as_array).map(|files| { files .iter() .map(|f| { let mb = |k: &str| finite(f.get(k)).map(|v| v * 1_048_576.0); json!({ "name": text(f.get("name"), 260), "allocated": opt(mb("allocatedMb")), "used": opt(mb("currentMb")), "peak": opt(mb("peakMb")), "temporary": f.get("temporary") == Some(&Value::Bool(true)) }) }) .collect() }); let page_total = pagefiles.as_ref().map(|p| p.iter().map(|f| f["allocated"].as_f64().unwrap_or(0.0)).sum::()); let page_used = pagefiles.as_ref().map(|p| p.iter().map(|f| f["used"].as_f64().unwrap_or(0.0)).sum::()); let disks: Vec = raw .get("disks") .and_then(Value::as_array) .cloned() .unwrap_or_default() .iter() .map(|d| { let used = match (finite(d.get("total")), finite(d.get("free"))) { (Some(t), Some(f)) => metric_value(t - f), _ => Value::Null, }; json!({ "name": text(d.get("name"), 16), "total": opt(finite(d.get("total"))), "available": opt(finite(d.get("available"))), "used": used, "format": text(d.get("format"), 16), "label": text(d.get("label"), 64) }) }) .collect(); let system_drive = raw.get("systemDrive").and_then(Value::as_str).filter(|s| !s.is_empty()).unwrap_or("C:").to_lowercase(); let chosen = disks.iter().find(|d| d["name"].as_str().map(|n| n.to_lowercase()) == Some(system_drive.clone())).or_else(|| disks.first()); let adapters: Vec = raw.get("adapters").and_then(Value::as_array).cloned().unwrap_or_default().into_iter().take(64).collect(); let default_index = adapters.iter().position(|a| a.get("gateway") == Some(&Value::Bool(true)) && a.get("state").and_then(Value::as_str) == Some("Up")); let memory_detail = match (counters, total) { (Some(c), Some(total)) if total != 0.0 => { let n = |k: &str| finite(c.get(k)).unwrap_or(0.0); let standby = n("standbyCore") + n("standbyNormal") + n("standbyReserve"); let in_use = (total - n("available") - n("modified")).max(0.0); json!({ "source": "Win32_PerfRawData_PerfOS_Memory", "slices": { "inUse": metric_value(in_use), "modified": c.get("modified").cloned().unwrap_or(Value::Null), "standby": metric_value(standby), "free": c.get("freeZero").cloned().unwrap_or(Value::Null) }, "available": c.get("available").cloned().unwrap_or(Value::Null), "systemCache": c.get("systemCacheResident").cloned().unwrap_or(Value::Null), "cacheBytes": c.get("cache").cloned().unwrap_or(Value::Null), "kernel": { "pagedPool": c.get("poolPaged").cloned().unwrap_or(Value::Null), "nonpagedPool": c.get("poolNonpaged").cloned().unwrap_or(Value::Null), "systemCode": c.get("systemCode").cloned().unwrap_or(Value::Null), "systemDriver": c.get("systemDriver").cloned().unwrap_or(Value::Null) }, "commit": { "committed": c.get("committed").cloned().unwrap_or(Value::Null), "limit": c.get("commitLimit").cloned().unwrap_or(Value::Null) }, "paging": { "pagesInPerSecond": rate(state.get("memoryCounters"), c, "pagesInput"), "pagesOutPerSecond": rate(state.get("memoryCounters"), c, "pagesOutput") } }) } _ => Value::Null, }; let build = text(s.get("build"), 40); let ubr = text(s.get("ubr"), 40); let kernel = [if build.is_empty() { String::new() } else { format!("build {build}{}", if ubr.is_empty() { String::new() } else { format!(".{ubr}") }) }, text(s.get("displayVersion"), 20)] .into_iter() .filter(|p| !p.is_empty()) .collect::>() .join(" "); let non_empty = |v: String| if v.is_empty() { Value::Null } else { Value::String(v) }; let interfaces: Vec = adapters .iter() .enumerate() .map(|(index, a)| { let rx = finite(a.get("rx")); let tx = finite(a.get("tx")); let rxp = finite(a.get("rxPackets")); let txp = finite(a.get("txPackets")); let bytes_reported = !(rx == Some(0.0) && tx == Some(0.0) && (rxp.unwrap_or(0.0) > 0.0 || txp.unwrap_or(0.0) > 0.0)); let state = a.get("state").and_then(Value::as_str).unwrap_or(""); json!({ "name": text(a.get("name"), 64), "description": text(a.get("description"), 120), "state": if state == "Up" { "up".to_string() } else { text(a.get("state"), 24).to_lowercase() }, "mac": text(a.get("mac"), 64), "rxBytes": if bytes_reported { opt(rx) } else { Value::Null }, "txBytes": if bytes_reported { opt(tx) } else { Value::Null }, "rxPackets": opt(rxp), "txPackets": opt(txp), "bytesUnavailableReason": if bytes_reported { Value::Null } else { json!("The network driver does not report transferred bytes (packets are counted).") }, "speedBitsPerSecond": finite(a.get("speed")).filter(|v| *v > 0.0).map(metric_value).unwrap_or(Value::Null), "addresses": a.get("addresses").and_then(Value::as_array).map(|list| list.iter().take(16).map(|x| Value::String(text(Some(x), 64))).collect::>()).unwrap_or_default(), "isDefault": Some(index) == default_index }) }) .collect(); let hostname = text(s.get("hostname"), 120); json!({ "os": "windows", "platform": "Windows", "kernel": non_empty(kernel), "osName": non_empty(text(s.get("caption"), 120)), "installationType": non_empty(text(s.get("installationType"), 40)), "hostname": if hostname.is_empty() { fallback_hostname.to_string() } else { hostname }, "uptimeSeconds": opt(finite(raw.get("uptimeSeconds"))), "load1": null, "loadAverageSupported": false, "processorQueueLength": opt(finite(raw.get("processorQueueLength"))), "processes": opt(finite(raw.get("processes"))), "memory": { "total": opt(total), "available": opt(available) }, "memoryDetail": memory_detail, "swap": { "kind": "pagefile", "total": opt(page_total), "used": opt(page_used), "free": match (page_total, page_used) { (Some(t), Some(u)) => metric_value((t - u).max(0.0)), _ => Value::Null } }, "pagefiles": pagefiles.map(Value::Array).unwrap_or(Value::Null), "disk": match chosen { Some(d) => json!({ "mount": d["name"], "total": d["total"], "used": d["used"], "available": d["available"] }), None => json!({ "mount": null, "total": null, "used": null, "available": null }), }, "disks": disks, "cpu": { "usagePercent": usage.map(|u| json!(u)).unwrap_or(Value::Null), "usagePending": usage.is_none() && raw.get("cpuRaw").map(|v| !v.is_null()).unwrap_or(false), "architecture": non_empty(text(s.get("architecture"), 32)), "model": non_empty(text(s.get("cpuModel"), 160)), "logicalCores": opt(finite(s.get("logical"))), "physicalCores": opt(finite(s.get("physical"))), "sockets": opt(finite(s.get("sockets"))), "coreTypes": [], "packages": windows_cpu_packages(raw) }, "gpus": windows_gpus(raw, s), "network": { "defaultInterface": default_index.map(|i| text(adapters[i].get("name"), 64)).unwrap_or_default(), "interfaces": interfaces }, "unavailable": raw.get("errors").and_then(Value::as_array).map(|e| e.iter().take(10).map(|x| Value::String(text(Some(x), 200))).collect::>()).unwrap_or_default() }) } pub async fn read_system_metrics(manager: &SessionManager, session_id: &str) -> CoreResult { let record = manager.require_connected(session_id)?; let platform = detect_platform(manager, &record, false).await?; let collected_at = chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); match platform["os"].as_str().unwrap_or("unknown") { "windows" => read_windows_metrics(manager, &record).await, "unknown" => Err(CoreError::new( "METRICS_PLATFORM_UNSUPPORTED", format!("The remote operating system could not be identified: {}.", platform["reason"].as_str().unwrap_or("no POSIX shell and no Windows PowerShell")), )), _ => { let output = manager.exec_fixed_default(&record, METRICS_COMMAND, Duration::from_secs(10)).await?; let mut metrics = parse_system_metrics_output(&output.stdout, &record.host); let object = metrics.as_object_mut().expect("object"); let mut ordered = Map::new(); ordered.insert("sessionId".into(), json!(record.session_id)); ordered.append(object); ordered.insert("collectedAt".into(), json!(collected_at)); Ok(Value::Object(ordered)) } } } async fn read_windows_metrics(manager: &SessionManager, record: &Arc) -> CoreResult { let has_static = record.metrics_state.lock().get("static").is_some(); let raw = run_windows_script(manager, record, WINDOWS_METRICS_SCRIPT, &json!({ "static": !has_static }), Duration::from_secs(30), 4 * 1024 * 1024, "METRICS_").await?; let mut state = record.metrics_state.lock(); if let Some(static_info) = raw.get("static").filter(|v| !v.is_null()) { state["static"] = static_info.clone(); } let static_info = state.get("static").cloned().unwrap_or(Value::Null); let metrics = normalize_windows_metrics(&raw, &static_info, &state, &record.host); if let Some(cpu) = raw.get("cpuRaw").filter(|v| !v.is_null()) { state["cpuRaw"] = cpu.clone(); } if let Some(counters) = raw.get("memoryCounters").filter(|v| !v.is_null()) { state["memoryCounters"] = counters.clone(); } let mut ordered = Map::new(); ordered.insert("sessionId".into(), json!(record.session_id)); if let Value::Object(mut object) = metrics { ordered.append(&mut object); } ordered.insert("collectedAt".into(), json!(chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true))); Ok(Value::Object(ordered)) } #[cfg(test)] mod tests { use super::*; #[test] fn linux_output_parsing() { let output = "platform=Linux\nkernel=6.8.0\nhostname=web\nuptimeSeconds=100\nload1=0.5\nmemoryTotal=1000\nmemoryAvailable=2000\nswapTotal=100\nswapFree=40\ndisk=1024:512:256\ncpuUsage=150\ncpuArchitecture=x86_64\ncpuModel=Intel|X\ncpuLogical=4\ncpuPhysical=2\ncpuSockets=1\ncpuCoreTypes=P-core:2,E-core:4\nnetworkDefault=eth0\nmeminfo=MemTotal|8000\nmeminfo=MemFree|1000\nmeminfo=Buffers|500\nmeminfo=Cached|2000\nmeminfo=SReclaimable|500\nswapArea=/dev/zram0|partition|1024|0|100\nnetworkInterface=eth0|UP|aa:bb|10|20|10.0.0.1/24,fe80::1/64\npageSize=4096\n"; let m = parse_system_metrics_output(output, "fallback"); assert_eq!(m["memory"]["available"], 1000); assert_eq!(m["swap"]["used"], 60); assert_eq!(m["cpu"]["usagePercent"], 100); assert_eq!(m["cpu"]["model"], "Intel X"); assert_eq!(m["cpu"]["coreTypes"][1]["name"], "E-core"); assert_eq!(m["memoryDetail"]["used"], 4000); assert_eq!(m["swapAreas"][0]["type"], "zram"); assert_eq!(m["network"]["interfaces"][0]["state"], "up"); assert_eq!(m["network"]["interfaces"][0]["isDefault"], true); assert_eq!(m["disk"]["used"], 512); } #[test] fn linux_cpu_packages_and_gpus() { let output = "cpuPackage=1|Xeon B|8|4|20.50|2100|\ncpuPackage=0|Xeon A|16|8|75.00|3400|61000\n\ gpu=0000:01:00.0|0x10de|0x2204|nvidia|NVIDIA Corporation GA102|||||||||\n\ gpu=0000:41:00.0|0x10de|0x2204|nvidia|NVIDIA Corporation GA102|||||||||\n\ gpu=0000:03:00.0|0x1002|0x73bf|amdgpu|AMD Navi 21|42|7|17163091968|1073741824|55000|120000000|250000000|1800000000|128\n\ gpu=0000:05:00.0|0x10de|0x1eb8|vfio-pci|NVIDIA Tesla T4|||||||||\n\ gpu=0000:07:00.0|0x1a03|0x2000|ast|ASPEED Graphics|||||||||\n\ nvidiaGpu=00000000:01:00.0|NVIDIA GeForce RTX 3090|88|40|24576|20000|71|310.5|350.00|1950|65|550.54\n\ nvidiaGpu=00000000:41:00.0|NVIDIA GeForce RTX 3090|[N/A]|[N/A]|24576|[N/A]|[N/A]|[Not Supported]|[N/A]|[N/A]|[N/A]|550.54\n"; let m = parse_system_metrics_output(output, "h"); let p = m["cpu"]["packages"].as_array().unwrap(); assert_eq!((p.len(), p[0]["id"].as_str(), p[1]["id"].as_str()), (2, Some("0"), Some("1"))); assert_eq!(p[0]["usagePercent"], 75); assert_eq!(p[0]["temperatureC"], 61); assert_eq!(p[1]["temperatureC"], Value::Null); let g = m["gpus"].as_array().unwrap(); assert_eq!(g.len(), 5); let (first, second, amd, passed, bmc) = (&g[0], &g[1], &g[2], &g[3], &g[4]); assert_eq!(first["name"], "NVIDIA GeForce RTX 3090"); assert_eq!((first["usagePercent"].as_f64(), first["powerW"].as_f64(), first["source"].as_str()), (Some(88.0), Some(310.5), Some("nvidia-smi"))); assert_eq!(first["memoryTotal"], 24576u64 * 1_048_576); assert_eq!(second["usagePercent"], Value::Null, "[N/A] is not 0 %"); assert_eq!(second["memoryUsed"], Value::Null); assert_eq!(amd["usagePercent"], 42); assert_eq!(amd["temperatureC"], 55); assert_eq!(amd["powerW"], 120); assert_eq!(amd["clockMhz"], 1800); assert_eq!(amd["fanPercent"].as_f64().unwrap().round(), 50.0); assert_eq!((passed["driver"].as_str(), passed["source"].as_str(), passed["usagePercent"].is_null()), (Some("vfio-pci"), Some("none"), true)); assert_eq!(bmc["name"], "ASPEED Graphics"); } #[test] fn emulated_gpus_get_a_name_when_lspci_has_none() { let output = "gpu=0000:00:02.0|0x1234|0x1111||Vendor 1234 Device 1111|||||||||\n\ gpu=0000:00:03.0|0x1234|0x1111|bochs-drm||||||||||\n\ gpu=0000:00:04.0|0x1b36|0x0100|qxl|Vendor 1b36 Device 0100|||||||||\n\ gpu=0000:00:05.0|0x1af4|0x1050|virtio-pci|Red Hat, Inc. Virtio 1.0 GPU|||||||||\n\ gpu=0000:00:06.0|0x10de|0xffff||Vendor 10de Device ffff|||||||||\n\ gpu=0000:00:07.0|0xabcd|0x0001||Vendor abcd Device 0001|||||||||\n"; let m = parse_system_metrics_output(output, "h"); let names: Vec<&str> = m["gpus"].as_array().unwrap().iter().map(|g| g["name"].as_str().unwrap()).collect(); assert_eq!(names, ["QEMU Standard VGA", "QEMU Standard VGA", "QEMU QXL VGA", "Red Hat, Inc. Virtio 1.0 GPU", "NVIDIA GPU", "Vendor abcd Device 0001"]); assert_eq!(m["gpus"][0]["vendorId"], "1234"); assert_eq!(m["gpus"][0]["usagePercent"], Value::Null, "a name does not invent metrics"); } #[test] fn linux_without_gpu_or_new_lines_is_empty() { let m = parse_system_metrics_output("platform=Linux\n", "h"); assert_eq!(m["gpus"], json!([])); assert_eq!(m["cpu"]["packages"], json!([])); } #[test] fn windows_gpus_join_nvidia_and_refuse_ambiguous_counters() { let controller = |name: &str, pnp: &str| json!({ "name": name, "pnp": pnp, "adapterRam": 4294967295u64, "driverVersion": "1" }); let raw = json!({ "nvidia": ["00000000:01:00.0, NVIDIA RTX A, 50, 10, 8192, 1024, 60, 100.0, 200.0, 1500, 40, 551.0", "00000000:02:00.0, NVIDIA RTX A, 5, 1, 8192, 512, 40, 20.0, 200.0, 300, 30, 551.0"], "gpuCounters": [{ "luid": "a", "usage": 9, "dedicated": 5 }, { "luid": "b", "usage": 1, "dedicated": 5 }], "cpuPackages": [{ "id": "CPU0", "name": "EPYC", "cores": 8, "logical": 16, "load": 30, "mhz": 2500 }, { "id": "CPU1", "name": "EPYC", "cores": 8, "logical": 16, "load": null, "mhz": 2500 }] }); let info = json!({ "gpus": [controller("NVIDIA RTX A", "PCI\\VEN_10DE&DEV_2204&X"), controller("NVIDIA RTX A", "PCI\\VEN_10DE&DEV_2204&Y")] }); let m = normalize_windows_metrics(&raw, &info, &json!({}), "h"); assert_eq!(m["gpus"][0]["usagePercent"], 50); assert_eq!(m["gpus"][1]["usagePercent"], 5); assert_eq!(m["gpus"][0]["vendorId"], "10de"); assert_eq!(m["cpu"]["packages"][1]["id"], "1"); assert_eq!(m["cpu"]["packages"][1]["usagePercent"], Value::Null); // One real adapter, one counter set: usable. Basic display adapter (1414) does not count. let one = json!({ "gpus": [controller("AMD X", "PCI\\VEN_1002&DEV_73BF"), controller("Microsoft Basic Display Adapter", "PCI\\VEN_1414&DEV_008C")] }); let raw = json!({ "gpuCounters": [{ "luid": "a", "usage": 33, "dedicated": 2048 }] }); let m = normalize_windows_metrics(&raw, &one, &json!({}), "h"); assert_eq!((m["gpus"][0]["usagePercent"].as_f64(), m["gpus"][0]["source"].as_str()), (Some(33.0), Some("windows-counters"))); assert_eq!(m["gpus"][0]["memoryTotal"], Value::Null, "the 32-bit ceiling is not a size"); // Two real adapters, two counter sets: no guessing. let two = json!({ "gpus": [controller("AMD X", "PCI\\VEN_1002&DEV_73BF"), controller("Intel Y", "PCI\\VEN_8086&DEV_46A6")] }); let raw = json!({ "gpuCounters": [{ "luid": "a", "usage": 33 }, { "luid": "b", "usage": 1 }] }); let m = normalize_windows_metrics(&raw, &two, &json!({}), "h"); assert_eq!(m["gpus"][0]["usagePercent"], Value::Null); } #[test] fn windows_cpu_from_raw_counters() { let previous = json!({"busyInverse": "1000", "time": "2000"}); let current = json!({"busyInverse": "1500", "time": "3000"}); assert_eq!(cpu_usage(Some(&previous), Some(¤t)), Some(50.0)); let raw = json!({"memoryTotal": 100.0, "memoryAvailable": 40.0, "cpuRaw": current, "pagefiles": [{"name": "C:\\pagefile.sys", "allocatedMb": 1, "currentMb": 0.5}], "disks": [{"name": "C:", "total": 10, "free": 4, "available": 4}], "systemDrive": "C:", "adapters": [{"name": "Ethernet", "state": "Up", "gateway": true, "rx": 0, "tx": 0, "rxPackets": 5, "txPackets": 1}]}); let m = normalize_windows_metrics(&raw, &json!({"build": "26100", "ubr": "1"}), &json!({"cpuRaw": previous}), "h"); assert_eq!(m["cpu"]["usagePercent"], 50.0); assert_eq!(m["swap"]["total"], 1_048_576); assert_eq!(m["disk"]["used"], 6); assert_eq!(m["network"]["interfaces"][0]["rxBytes"], Value::Null); assert_eq!(m["kernel"], "build 26100.1"); assert_eq!(m["load1"], Value::Null); } }