feat: add nvidia-smi pmon integration to detect NVENC/NVDEC processes and display GPU process activity with auto-refresh status
Add graphics_process_log_state global to track GPU process changes. Query nvidia-smi pmon with 1-count sample to detect video encode/decode processes missed by compute-apps query. Parse pmon output for SM/ENC/DEC utilization percentages and process types. Merge pmon results with compute-apps by PID, preferring pmon data when available. Log GPU process detection with PID/name
This commit is contained in:
@@ -88,6 +88,7 @@ raid_refresh_lock = threading.Lock()
|
|||||||
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0}
|
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0}
|
||||||
raid_log_state = None
|
raid_log_state = None
|
||||||
graphics_log_state = None
|
graphics_log_state = None
|
||||||
|
graphics_process_log_state = None
|
||||||
hardware_log_once = set()
|
hardware_log_once = set()
|
||||||
|
|
||||||
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
|
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
|
||||||
@@ -357,7 +358,7 @@ def io_pressure():
|
|||||||
|
|
||||||
def graphics_snapshot():
|
def graphics_snapshot():
|
||||||
"""Read NVIDIA GPU details with short, non-blocking subprocess timeouts."""
|
"""Read NVIDIA GPU details with short, non-blocking subprocess timeouts."""
|
||||||
global graphics_log_state
|
global graphics_log_state, graphics_process_log_state
|
||||||
query = "index,name,uuid,serial,pci.bus_id,driver_version,pstate,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.used,memory.free,power.draw,power.limit,fan.speed,clocks.current.graphics,clocks.current.memory,clocks.max.graphics,clocks.max.memory,compute_mode"
|
query = "index,name,uuid,serial,pci.bus_id,driver_version,pstate,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.used,memory.free,power.draw,power.limit,fan.speed,clocks.current.graphics,clocks.current.memory,clocks.max.graphics,clocks.max.memory,compute_mode"
|
||||||
try:
|
try:
|
||||||
result = subprocess.run([NVIDIA_SMI_BIN, f"--query-gpu={query}", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
result = subprocess.run([NVIDIA_SMI_BIN, f"--query-gpu={query}", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
||||||
@@ -388,15 +389,49 @@ def graphics_snapshot():
|
|||||||
try: gpu[key] = round(float(gpu[key]), 1)
|
try: gpu[key] = round(float(gpu[key]), 1)
|
||||||
except (TypeError, ValueError): gpu[key] = None
|
except (TypeError, ValueError): gpu[key] = None
|
||||||
gpus.append(gpu)
|
gpus.append(gpu)
|
||||||
processes = []
|
processes_by_pid = {}
|
||||||
try:
|
try:
|
||||||
proc = subprocess.run([NVIDIA_SMI_BIN, "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
proc = subprocess.run([NVIDIA_SMI_BIN, "--query-compute-apps=pid,process_name,used_gpu_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, timeout=5, check=False)
|
||||||
for line in proc.stdout.splitlines():
|
if proc.returncode == 0:
|
||||||
values = [value.strip() for value in line.split(",")]
|
for line in proc.stdout.splitlines():
|
||||||
if len(values) >= 3:
|
values = [value.strip() for value in line.split(",")]
|
||||||
processes.append({"pid": values[0], "name": values[1], "memory": values[2]})
|
if len(values) >= 3 and values[0].isdigit():
|
||||||
|
processes_by_pid[values[0]] = {"pid": values[0], "name": values[1], "memory": values[2], "type": "compute"}
|
||||||
except (OSError, subprocess.SubprocessError):
|
except (OSError, subprocess.SubprocessError):
|
||||||
pass
|
pass
|
||||||
|
pmon_error = ""
|
||||||
|
try:
|
||||||
|
# Compute-apps misses NVENC/NVDEC. pmon also reports video/graphics
|
||||||
|
# processes such as Jellyfin while a hardware transcode is running.
|
||||||
|
pmon = subprocess.run([NVIDIA_SMI_BIN, "pmon", "-c", "1", "-s", "um"], capture_output=True, text=True, timeout=5, check=False)
|
||||||
|
if pmon.returncode == 0:
|
||||||
|
for line in pmon.stdout.splitlines():
|
||||||
|
values = line.split()
|
||||||
|
if not values or values[0].startswith("#") or len(values) < 8 or not values[1].isdigit():
|
||||||
|
continue
|
||||||
|
pid, process_type = values[1], values[2]
|
||||||
|
command = " ".join(values[7:]) or "GPU-Prozess"
|
||||||
|
processes_by_pid[pid] = {
|
||||||
|
"pid": pid,
|
||||||
|
"name": command,
|
||||||
|
"memory": f"SM {values[3]}% · ENC {values[5]}% · DEC {values[6]}%",
|
||||||
|
"type": process_type,
|
||||||
|
"sm": values[3], "enc": values[5], "dec": values[6],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
pmon_error = " ".join((pmon.stderr or "pmon ohne Ausgabe").split())[:180]
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
pmon_error = error_summary(exc)
|
||||||
|
processes = list(processes_by_pid.values())
|
||||||
|
process_signature = tuple((item["pid"], item["name"], item.get("type")) for item in processes)
|
||||||
|
if process_signature != graphics_process_log_state:
|
||||||
|
if processes:
|
||||||
|
log.info("GPU-Prozesse erkannt: %s", ", ".join(f"{item['name']} (PID {item['pid']})" for item in processes))
|
||||||
|
elif pmon_error:
|
||||||
|
log.warning("GPU-Prozessabfrage fehlgeschlagen: %s", pmon_error)
|
||||||
|
else:
|
||||||
|
log.info("Keine GPU-Prozesse erkannt")
|
||||||
|
graphics_process_log_state = process_signature
|
||||||
signature = ("ok", tuple((gpu.get("index"), gpu.get("name"), gpu.get("temperature.gpu"), gpu.get("utilization.gpu")) for gpu in gpus))
|
signature = ("ok", tuple((gpu.get("index"), gpu.get("name"), gpu.get("temperature.gpu"), gpu.get("utilization.gpu")) for gpu in gpus))
|
||||||
if signature != graphics_log_state:
|
if signature != graphics_log_state:
|
||||||
log.info("GPU-Status verfügbar: %s NVIDIA-GPU(s) via %s", len(gpus), NVIDIA_SMI_BIN)
|
log.info("GPU-Status verfügbar: %s NVIDIA-GPU(s) via %s", len(gpus), NVIDIA_SMI_BIN)
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user