feat: add independent background monitors for RAID/IO/GPU with configurable refresh intervals to eliminate blocking queries
Add RAID_REFRESH_SECONDS/IO_REFRESH_SECONDS/GPU_REFRESH_SECONDS environment variables with default 5 seconds and minimum 2 seconds. Add graphics_cache_lock and graphics_cache global for thread-safe GPU data access. Add io/io_at fields to datastore_cache. Extract refresh_io_status() to update I/O pressure cache. Add refresh_gpu_status() to update graphics_cache. Implement hardware
This commit is contained in:
@@ -90,10 +90,14 @@ SCAN_WORKERS=4
|
|||||||
REQUEST_TIMEOUT=20
|
REQUEST_TIMEOUT=20
|
||||||
DB_PATH=/data/cache.db
|
DB_PATH=/data/cache.db
|
||||||
SYNC_INTERVAL_SECONDS=1800
|
SYNC_INTERVAL_SECONDS=1800
|
||||||
|
RAID_REFRESH_SECONDS=5
|
||||||
|
IO_REFRESH_SECONDS=5
|
||||||
|
GPU_REFRESH_SECONDS=5
|
||||||
```
|
```
|
||||||
|
|
||||||
`SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last.
|
`SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last.
|
||||||
`SYNC_INTERVAL_SECONDS` steuert den serverseitigen Sync-Takt; `1800` entspricht 30 Minuten, `18000` fünf Stunden und `0` deaktiviert die periodische Wiederholung (der Start-Sync bleibt aktiv).
|
`SYNC_INTERVAL_SECONDS` steuert den serverseitigen Sync-Takt; `1800` entspricht 30 Minuten, `18000` fünf Stunden und `0` deaktiviert die periodische Wiederholung (der Start-Sync bleibt aktiv).
|
||||||
|
RAID, I/O-Pressure und GPU werden unabhängig vom Browser durch eigene Hintergrund-Monitoren aktualisiert. Die Werte werden aus dem Server-Cache geliefert; die drei Intervalle können separat angepasst werden.
|
||||||
|
|
||||||
## Datastore, RAID und SMART
|
## Datastore, RAID und SMART
|
||||||
|
|
||||||
|
|||||||
@@ -49,6 +49,9 @@ SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true"
|
|||||||
LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log")
|
LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log")
|
||||||
LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6")))
|
LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6")))
|
||||||
SYNC_INTERVAL_SECONDS = max(0, int(os.environ.get("SYNC_INTERVAL_SECONDS", "1800")))
|
SYNC_INTERVAL_SECONDS = max(0, int(os.environ.get("SYNC_INTERVAL_SECONDS", "1800")))
|
||||||
|
RAID_REFRESH_SECONDS = max(2, int(os.environ.get("RAID_REFRESH_SECONDS", "5")))
|
||||||
|
IO_REFRESH_SECONDS = max(2, int(os.environ.get("IO_REFRESH_SECONDS", "5")))
|
||||||
|
GPU_REFRESH_SECONDS = max(2, int(os.environ.get("GPU_REFRESH_SECONDS", "5")))
|
||||||
|
|
||||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
|
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
|
||||||
log = logging.getLogger("media-max")
|
log = logging.getLogger("media-max")
|
||||||
@@ -86,10 +89,12 @@ service_check_executor = ThreadPoolExecutor(max_workers=16)
|
|||||||
datastore_cache_lock = threading.Lock()
|
datastore_cache_lock = threading.Lock()
|
||||||
datastore_refresh_lock = threading.Lock()
|
datastore_refresh_lock = threading.Lock()
|
||||||
raid_refresh_lock = threading.Lock()
|
raid_refresh_lock = threading.Lock()
|
||||||
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0}
|
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0, "io": None, "io_at": 0.0}
|
||||||
raid_log_state = None
|
raid_log_state = None
|
||||||
graphics_log_state = None
|
graphics_log_state = None
|
||||||
graphics_process_log_state = None
|
graphics_process_log_state = None
|
||||||
|
graphics_cache_lock = threading.Lock()
|
||||||
|
graphics_cache = None
|
||||||
hardware_log_once = set()
|
hardware_log_once = set()
|
||||||
library_cache_lock = threading.Lock()
|
library_cache_lock = threading.Lock()
|
||||||
library_cache = {"rows": [], "series": [], "updated_at": 0.0, "error": None, "ready": False}
|
library_cache = {"rows": [], "series": [], "updated_at": 0.0, "error": None, "ready": False}
|
||||||
@@ -657,6 +662,36 @@ def refresh_raid_status():
|
|||||||
finally:
|
finally:
|
||||||
raid_refresh_lock.release()
|
raid_refresh_lock.release()
|
||||||
|
|
||||||
|
def refresh_io_status():
|
||||||
|
current = io_pressure()
|
||||||
|
with datastore_cache_lock:
|
||||||
|
datastore_cache["io"] = current
|
||||||
|
datastore_cache["io_at"] = time.time()
|
||||||
|
|
||||||
|
def _hardware_loop(name, interval, refresh):
|
||||||
|
log.info("%s-Monitor gestartet (Intervall: %ss)", name, interval)
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
refresh()
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("%s-Monitor fehlgeschlagen: %s", name, error_summary(exc))
|
||||||
|
time.sleep(interval)
|
||||||
|
|
||||||
|
def refresh_gpu_status():
|
||||||
|
global graphics_cache
|
||||||
|
current = graphics_snapshot()
|
||||||
|
with graphics_cache_lock:
|
||||||
|
graphics_cache = current
|
||||||
|
|
||||||
|
def hardware_status_monitor():
|
||||||
|
"""Run slow hardware probes independently and in parallel."""
|
||||||
|
for name, interval, refresh in (
|
||||||
|
("RAID", RAID_REFRESH_SECONDS, refresh_raid_status),
|
||||||
|
("I/O", IO_REFRESH_SECONDS, refresh_io_status),
|
||||||
|
("GPU", GPU_REFRESH_SECONDS, refresh_gpu_status),
|
||||||
|
):
|
||||||
|
threading.Thread(target=_hardware_loop, args=(name, interval, refresh), name=f"{name.lower()}-monitor", daemon=True).start()
|
||||||
|
|
||||||
def datastore_snapshot():
|
def datastore_snapshot():
|
||||||
now = time.time()
|
now = time.time()
|
||||||
with datastore_cache_lock:
|
with datastore_cache_lock:
|
||||||
@@ -670,13 +705,11 @@ def datastore_snapshot():
|
|||||||
storage = datastore_cache["snapshot"]
|
storage = datastore_cache["snapshot"]
|
||||||
with datastore_cache_lock:
|
with datastore_cache_lock:
|
||||||
current_raid = datastore_cache.get("raid")
|
current_raid = datastore_cache.get("raid")
|
||||||
raid_age = now - datastore_cache.get("raid_at", 0)
|
|
||||||
if current_raid is None:
|
if current_raid is None:
|
||||||
threading.Thread(target=refresh_raid_status, name="raid-refresh", daemon=True).start()
|
|
||||||
current_raid = {"device": RAID_DEVICE, "status": "unknown", "label": "WIRD GELADEN", "error": "RAID-Status wird geladen"}
|
current_raid = {"device": RAID_DEVICE, "status": "unknown", "label": "WIRD GELADEN", "error": "RAID-Status wird geladen"}
|
||||||
elif raid_age >= 5:
|
with datastore_cache_lock:
|
||||||
threading.Thread(target=refresh_raid_status, name="raid-refresh", daemon=True).start()
|
current_io = datastore_cache.get("io") or {"available": False, "some": {}, "full": {}}
|
||||||
return {"storage": storage, "raid": current_raid, "io_pressure": io_pressure(), "cache_seconds": STORAGE_CACHE_SECONDS}
|
return {"storage": storage, "raid": current_raid, "io_pressure": current_io, "cache_seconds": STORAGE_CACHE_SECONDS}
|
||||||
|
|
||||||
def cpu_usage_percent():
|
def cpu_usage_percent():
|
||||||
global previous_cpu
|
global previous_cpu
|
||||||
@@ -1038,6 +1071,7 @@ def background_library_sync():
|
|||||||
time.sleep(SYNC_INTERVAL_SECONDS)
|
time.sleep(SYNC_INTERVAL_SECONDS)
|
||||||
|
|
||||||
threading.Thread(target=background_library_sync, name="library-sync", daemon=True).start()
|
threading.Thread(target=background_library_sync, name="library-sync", daemon=True).start()
|
||||||
|
threading.Thread(target=hardware_status_monitor, name="hardware-monitor", daemon=True).start()
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
def index():
|
def index():
|
||||||
@@ -1123,11 +1157,9 @@ def datastore():
|
|||||||
|
|
||||||
@app.get("/api/graphics")
|
@app.get("/api/graphics")
|
||||||
def graphics():
|
def graphics():
|
||||||
try:
|
with graphics_cache_lock:
|
||||||
return jsonify(graphics_snapshot())
|
snapshot = graphics_cache
|
||||||
except Exception as exc:
|
return jsonify(snapshot or {"available": False, "gpus": [], "processes": [], "error": "GPU-Status wird geladen"})
|
||||||
log.error("GPU-Status konnte nicht abgefragt werden: %s", error_summary(exc))
|
|
||||||
return jsonify({"available": False, "gpus": [], "processes": [], "error": "GPU-Status momentan nicht verfügbar"})
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
def health(): return jsonify({"ok":True, "radarr_url":RADARR_URL, "sonarr_enabled":bool(SONARR_URL and SONARR_API_KEY), "sab_enabled":bool(SAB_URL and SAB_API_KEY)})
|
def health(): return jsonify({"ok":True, "radarr_url":RADARR_URL, "sonarr_enabled":bool(SONARR_URL and SONARR_API_KEY), "sab_enabled":bool(SAB_URL and SAB_API_KEY)})
|
||||||
|
|||||||
@@ -22,6 +22,9 @@ services:
|
|||||||
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
||||||
# Serverseitiger Bibliotheks-Sync; 1800 = alle 30 Minuten, 0 = nur beim Start.
|
# Serverseitiger Bibliotheks-Sync; 1800 = alle 30 Minuten, 0 = nur beim Start.
|
||||||
SYNC_INTERVAL_SECONDS: ${SYNC_INTERVAL_SECONDS:-1800}
|
SYNC_INTERVAL_SECONDS: ${SYNC_INTERVAL_SECONDS:-1800}
|
||||||
|
RAID_REFRESH_SECONDS: ${RAID_REFRESH_SECONDS:-5}
|
||||||
|
IO_REFRESH_SECONDS: ${IO_REFRESH_SECONDS:-5}
|
||||||
|
GPU_REFRESH_SECONDS: ${GPU_REFRESH_SECONDS:-5}
|
||||||
volumes:
|
volumes:
|
||||||
# Host-Pfad deiner Film-Library -> interner Dashboard-Pfad
|
# Host-Pfad deiner Film-Library -> interner Dashboard-Pfad
|
||||||
- ${FILM_MEDIA_VOLUME}:/media/filme:ro
|
- ${FILM_MEDIA_VOLUME}:/media/filme:ro
|
||||||
|
|||||||
Reference in New Issue
Block a user