feat: add nvidia-smi pmon integration to detect NVENC/NVDEC processes and display GPU process activity with auto-refresh status
Add graphics_process_log_state global to track GPU process changes. Query nvidia-smi pmon with 1-count sample to detect video encode/decode processes missed by compute-apps query. Parse pmon output for SM/ENC/DEC utilization percentages and process types. Merge pmon results with compute-apps by PID, preferring pmon data when available. Log GPU process detection with PID/name
This commit is contained in:
@@ -88,6 +88,7 @@ raid_refresh_lock = threading.Lock()
|
||||
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0}
|
||||
raid_log_state = None
|
||||
graphics_log_state = None
|
||||
graphics_process_log_state = None
|
||||
hardware_log_once = set()
|
||||
|
||||
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
|
||||
@@ -357,7 +358,7 @@ def io_pressure():
|
||||
|
||||
def graphics_snapshot():
|
||||
"""Read NVIDIA GPU details with short, non-blocking subprocess timeouts."""
|
||||
global graphics_log_state
|
||||
global graphics_log_state, graphics_process_log_state
|
||||
query = "index,name,uuid,serial,pci.bus_id,driver_version,pstate,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.used,memory.free,power.draw,power.limit,fan.speed,clocks.current.graphics,clocks.current.memory,clocks.max.graphics,clocks.max.memory,compute_mode"
|
||||
try:
|
||||
result = subprocess.run([NVIDIA_SMI_BIN, f"--query-gpu={query}", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
||||
@@ -388,15 +389,49 @@ def graphics_snapshot():
|
||||
try: gpu[key] = round(float(gpu[key]), 1)
|
||||
except (TypeError, ValueError): gpu[key] = None
|
||||
gpus.append(gpu)
|
||||
processes = []
|
||||
processes_by_pid = {}
|
||||
try:
|
||||
proc = subprocess.run([NVIDIA_SMI_BIN, "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
||||
for line in proc.stdout.splitlines():
|
||||
values = [value.strip() for value in line.split(",")]
|
||||
if len(values) >= 3:
|
||||
processes.append({"pid": values[0], "name": values[1], "memory": values[2]})
|
||||
if proc.returncode == 0:
|
||||
for line in proc.stdout.splitlines():
|
||||
values = [value.strip() for value in line.split(",")]
|
||||
if len(values) >= 3 and values[0].isdigit():
|
||||
processes_by_pid[values[0]] = {"pid": values[0], "name": values[1], "memory": values[2], "type": "compute"}
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
pass
|
||||
pmon_error = ""
|
||||
try:
|
||||
# Compute-apps misses NVENC/NVDEC. pmon also reports video/graphics
|
||||
# processes such as Jellyfin while a hardware transcode is running.
|
||||
pmon = subprocess.run([NVIDIA_SMI_BIN, "pmon", "-c", "1", "-s", "um"], capture_output=True, text=True, timeout=5, check=False)
|
||||
if pmon.returncode == 0:
|
||||
for line in pmon.stdout.splitlines():
|
||||
values = line.split()
|
||||
if not values or values[0].startswith("#") or len(values) < 8 or not values[1].isdigit():
|
||||
continue
|
||||
pid, process_type = values[1], values[2]
|
||||
command = " ".join(values[7:]) or "GPU-Prozess"
|
||||
processes_by_pid[pid] = {
|
||||
"pid": pid,
|
||||
"name": command,
|
||||
"memory": f"SM {values[3]}% · ENC {values[5]}% · DEC {values[6]}%",
|
||||
"type": process_type,
|
||||
"sm": values[3], "enc": values[5], "dec": values[6],
|
||||
}
|
||||
else:
|
||||
pmon_error = " ".join((pmon.stderr or "pmon ohne Ausgabe").split())[:180]
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
pmon_error = error_summary(exc)
|
||||
processes = list(processes_by_pid.values())
|
||||
process_signature = tuple((item["pid"], item["name"], item.get("type")) for item in processes)
|
||||
if process_signature != graphics_process_log_state:
|
||||
if processes:
|
||||
log.info("GPU-Prozesse erkannt: %s", ", ".join(f"{item['name']} (PID {item['pid']})" for item in processes))
|
||||
elif pmon_error:
|
||||
log.warning("GPU-Prozessabfrage fehlgeschlagen: %s", pmon_error)
|
||||
else:
|
||||
log.info("Keine GPU-Prozesse erkannt")
|
||||
graphics_process_log_state = process_signature
|
||||
signature = ("ok", tuple((gpu.get("index"), gpu.get("name"), gpu.get("temperature.gpu"), gpu.get("utilization.gpu")) for gpu in gpus))
|
||||
if signature != graphics_log_state:
|
||||
log.info("GPU-Status verfügbar: %s NVIDIA-GPU(s) via %s", len(gpus), NVIDIA_SMI_BIN)
|
||||
|
||||
Reference in New Issue
Block a user