feat: add scalable CPU and GPU telemetry

dashboard/plugin_api.py

@@ -73,6 +73,8 @@ _jobs: dict[str, dict[str, Any]] = {}
 _jobs_lock = threading.Lock()
 _model_loads: dict[str, dict[str, Any]] = {}
 _model_loads_lock = threading.Lock()
+_cpu_previous: dict[str, tuple[int, int]] = {}
+_cpu_previous_lock = threading.Lock()
 _chat_requests: dict[str, dict[str, Any]] = {}
 _chat_requests_lock = threading.Lock()
 _catalog_lock = threading.Lock()
@@ -481,9 +483,51 @@ def _host_ram_gib() -> float | None:
     return round(total / (1024 ** 3), 1) if total else None
 
 
+def _cpu_snapshot() -> dict[str, Any]:
+    """Return total and per-core CPU usage from Linux procfs counters."""
+    counters: dict[str, tuple[int, int]] = {}
+    try:
+        for line in Path("/proc/stat").read_text(encoding="utf-8").splitlines():
+            parts = line.split()
+            if not parts or not re.fullmatch(r"cpu(?:[0-9]+)?", parts[0]) or len(parts) < 5:
+                continue
+            values = [int(value) for value in parts[1:]]
+            total = sum(values)
+            idle = values[3] + (values[4] if len(values) > 4 else 0)
+            counters[parts[0]] = (total, idle)
+    except (OSError, ValueError):
+        return {"detected": False, "count": 0, "usage_percent": None, "cores": [], "load_average": []}
+    with _cpu_previous_lock:
+        previous = dict(_cpu_previous)
+        _cpu_previous.clear()
+        _cpu_previous.update(counters)
+    def usage(name: str) -> float:
+        current_total, current_idle = counters[name]
+        previous_values = previous.get(name)
+        if not previous_values:
+            return 0.0
+        previous_total, previous_idle = previous_values
+        delta_total = current_total - previous_total
+        delta_idle = current_idle - previous_idle
+        return round(max(0.0, min(100.0, (delta_total - delta_idle) * 100 / delta_total)), 1) if delta_total else 0.0
+    core_names = sorted((name for name in counters if name != "cpu"), key=lambda name: int(name[3:]))
+    cores = [{"id": int(name[3:]), "name": name, "usage_percent": usage(name)} for name in core_names]
+    try:
+        load_average = [float(value) for value in Path("/proc/loadavg").read_text(encoding="utf-8").split()[:3]]
+    except (OSError, ValueError):
+        load_average = []
+    return {
+        "detected": "cpu" in counters,
+        "count": len(cores),
+        "usage_percent": usage("cpu") if "cpu" in counters else None,
+        "cores": cores,
+        "load_average": load_average,
+    }
+
+
 def _gpu_snapshot() -> dict[str, Any]:
-    """Return NVIDIA GPU telemetry when available, without requiring CUDA."""
-    query = "name,memory.total,memory.used,memory.free"
+    """Return per-NVIDIA-GPU telemetry when available, without requiring CUDA."""
+    query = "index,name,memory.total,memory.used,memory.free,utilization.gpu,temperature.gpu,power.draw,power.limit"
     try:
         result = subprocess.run(
             ["nvidia-smi", f"--query-gpu={query}", "--format=csv,noheader,nounits"],
@@ -498,22 +542,38 @@ def _gpu_snapshot() -> dict[str, Any]:
         gpus = []
         for line in result.stdout.splitlines():
             parts = [part.strip() for part in line.split(",")]
-            if len(parts) != 4:
+            if len(parts) != 9:
                 continue
+            def number(value: str) -> float | None:
+                try:
+                    return None if value.upper() in {"N/A", "NA", "[N/A]"} else float(value)
+                except ValueError:
+                    return None
             try:
-                total, used, free = (int(float(value)) * 1024 * 1024 for value in parts[1:])
+                total, used, free = (int(float(value)) * 1024 * 1024 for value in parts[2:5])
             except ValueError:
                 continue
-            gpus.append({"name": parts[0], "total_bytes": total, "used_bytes": used, "free_bytes": free})
+            gpus.append({
+                "index": int(parts[0]) if parts[0].isdigit() else len(gpus),
+                "name": parts[1],
+                "total_bytes": total,
+                "used_bytes": used,
+                "free_bytes": free,
+                "utilization_percent": number(parts[5]),
+                "temperature_c": number(parts[6]),
+                "power_watts": number(parts[7]),
+                "power_limit_watts": number(parts[8]),
+            })
         if gpus:
-            return {"detected": True, "telemetry_available": True, "gpus": gpus}
+            utilization_values = [item["utilization_percent"] for item in gpus if item["utilization_percent"] is not None]
+            return {"detected": True, "telemetry_available": True, "count": len(gpus), "utilization_percent": round(sum(utilization_values) / len(utilization_values), 1) if utilization_values else None, "gpus": gpus}
     nvidia_present = False
     for vendor in Path("/sys/class/drm").glob("card*/device/vendor"):
         try:
             nvidia_present = nvidia_present or vendor.read_text().strip().lower() == "0x10de"
         except OSError:
             continue
-    return {"detected": nvidia_present, "telemetry_available": False, "gpus": []}
+    return {"detected": nvidia_present, "telemetry_available": False, "count": 0, "utilization_percent": None, "gpus": []}
 
 
 def _runtime_snapshot() -> dict[str, Any]:
@@ -548,6 +608,7 @@ def _runtime_snapshot() -> dict[str, Any]:
             "permanent": True,
         })
     gpu = _gpu_snapshot()
+    cpu = _cpu_snapshot()
     ollama_model_bytes = sum(int(row.get("total_bytes") or 0) for row in model_memory)
     ollama_model_vram_bytes = sum(int(row.get("gpu_bytes") or 0) for row in model_memory)
     model_loading = []
@@ -572,6 +633,7 @@ def _runtime_snapshot() -> dict[str, Any]:
         "ollama_model_bytes": ollama_model_bytes,
         "ollama_model_vram_bytes": ollama_model_vram_bytes,
         "ollama_target_model_bytes": ollama_target_model_bytes,
+        "cpu": cpu,
         "gpu": gpu,
     }