From 5ddf506f20b0a5f5a08922f4ff4e75b20742afc1 Mon Sep 17 00:00:00 2001 From: nessi Date: Sun, 16 Aug 2026 11:55:45 +0200 Subject: [PATCH] feat: add datastore view with storage categories, RAID status monitoring, and live recovery tracking Add scan_datastore_storage() to walk DATASTORE_PATH and categorize media by directory structure (movies/series/anime/kdrama/downloads/other) with size/file counts. Add raid_status() to parse /proc/mdstat and mdadm --detail for RAID level, active/failed devices, and recovery progress. Add /api/datastore endpoint with 15-minute cache for storage scans and live RAID updates. Install mdadm/util-linux/ --- .env.example | 6 ++ .gitignore | 2 + Dockerfile | 2 +- README.md | 8 ++ app.py | 172 +++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 6 ++ templates/index.html | 7 +- 7 files changed, 201 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 32517cb..e255670 100644 --- a/.env.example +++ b/.env.example @@ -31,4 +31,10 @@ SERVICE_CHECK_TIMEOUT=3 SERVICE_REFRESH_SECONDS=30 DB_PATH=/data/cache.db +DATASTORE_PATH=/nesflix +DATASTORE_HOST_PATH=/nesflix +RAID_DEVICE=/dev/md127 +STORAGE_CACHE_SECONDS=900 +# Optional: SMART-Abfragen aktivieren (benötigt Zugriff auf Host-Devices) +SMART_ENABLED=false LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore index b6bde21..c886b93 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ data/ __pycache__/ *.pyc +AGENT.md + diff --git a/Dockerfile b/Dockerfile index 9d2e47f..c9e1c96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.13-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends ffmpeg \ + && apt-get install -y --no-install-recommends ffmpeg mdadm util-linux smartmontools \ && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY requirements.txt . diff --git a/README.md b/README.md index 2fa1719..9b90243 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # Media Max +## Datastore + +Der Datastore-Tab zeigt den read-only Speicherstatus von `/nesflix`, Medienkategorien +und den Linux-Software-RAID-Status. Größen-Scans werden standardmäßig 15 Minuten +gecacht; RAID-/Recovery-Werte aktualisieren sich live. Konfigurierbar sind +`DATASTORE_PATH`, `DATASTORE_HOST_PATH`, `RAID_DEVICE`, `STORAGE_CACHE_SECONDS` und +optional `SMART_ENABLED=true` für SMART-Healthwerte. + ## Wichtig: Path-Mapping Radarr kann seine Filme z.B. unter `/data/filme` sehen, während der Host sie unter diff --git a/app.py b/app.py index 11ec897..8dcd795 100644 --- a/app.py +++ b/app.py @@ -38,6 +38,11 @@ SERVICE_CHECK_TIMEOUT = float(os.environ.get("SERVICE_CHECK_TIMEOUT", "3")) SERVICE_REFRESH_SECONDS = int(os.environ.get("SERVICE_REFRESH_SECONDS", "30")) DB_PATH = os.environ.get("DB_PATH", "/data/cache.db") REQUEST_TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT", "20")) +DATASTORE_PATH = os.environ.get("DATASTORE_PATH", "/nesflix").rstrip("/") or "/" +RAID_DEVICE = os.environ.get("RAID_DEVICE", "/dev/md127") +HOST_DEVICE_PATH = os.environ.get("HOST_DEVICE_PATH", "/host-dev") +STORAGE_CACHE_SECONDS = int(os.environ.get("STORAGE_CACHE_SECONDS", "900")) +SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"} logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s") log = logging.getLogger("media-max") @@ -50,6 +55,8 @@ system_lock = threading.Lock() previous_cpu = None service_status_lock = threading.Lock() service_check_executor = ThreadPoolExecutor(max_workers=16) +datastore_cache_lock = threading.Lock() +datastore_cache = {"stored_at": 0.0, "snapshot": None} LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"} FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"} @@ -160,6 +167,163 @@ def format_bytes(size): return f"{value:.1f} {unit}" value /= 1024 +def _normalized_path_part(value): + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + +def _datastore_category(relative_path, category_roots): + parts = [part for part in Path(relative_path).parts if part not in (".", "")] + normalized = {_normalized_path_part(part) for part in parts} + if normalized & {"download", "downloads", "sabnzbd", "nzb"}: + return "downloads" + if normalized & {"anime"}: + return "anime" + if normalized & {"kdrama", "kdramas", "kdramen", "korean", "koreanischeserien"}: + return "kdrama" + if parts and _normalized_path_part(parts[0]) in category_roots["movies"]: + return "movies" + if parts and _normalized_path_part(parts[0]) in category_roots["series"]: + return "series" + return "other" + +def scan_datastore_storage(): + root = Path(DATASTORE_PATH) + categories = {key: {"key": key, "label": label, "bytes": 0, "files": 0} for key, label in ( + ("movies", "Filme"), ("series", "Serien"), ("anime", "Anime"), + ("kdrama", "K-Dramen"), ("downloads", "Downloads"), ("other", "Sonstiges") + )} + if not root.exists() or not root.is_dir(): + return {"error": f"Datastore {DATASTORE_PATH} ist nicht verfügbar.", "path": DATASTORE_PATH, "categories": list(categories.values())} + category_roots = { + "movies": {"filme", "film", "movies", "movie"}, + "series": {"serien", "serie", "series", "tv", "shows"}, + } + try: + for current, directories, files in os.walk(root, followlinks=False): + directories[:] = [directory for directory in directories if not os.path.islink(os.path.join(current, directory))] + relative = os.path.relpath(current, root) + for filename in files: + full_path = os.path.join(current, filename) + if os.path.islink(full_path): + continue + try: + size = os.path.getsize(full_path) + except OSError: + continue + category = categories[_datastore_category(relative, category_roots)] + category["bytes"] += size + category["files"] += 1 + usage = shutil.disk_usage(root) + except OSError as exc: + return {"error": f"Datastore konnte nicht gelesen werden: {exc}", "path": DATASTORE_PATH, "categories": list(categories.values())} + used = usage.used + values = [] + for category in categories.values(): + category["size"] = format_bytes(category["bytes"]) + category["percent"] = round(category["bytes"] / used * 100, 1) if used else 0 + values.append(category) + return {"error": None, "path": DATASTORE_PATH, "total_bytes": usage.total, "used_bytes": usage.used, + "free_bytes": usage.free, "total": format_bytes(usage.total), "used": format_bytes(usage.used), + "free": format_bytes(usage.free), "percent": round(usage.used / usage.total * 100, 1) if usage.total else 0, + "categories": values, "scanned_at": time.time()} + +def _read_command(command, timeout=4): + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False) + return result.stdout, result.stderr + except (OSError, subprocess.SubprocessError) as exc: + return "", str(exc) + +def _flatten_lsblk(nodes): + result = [] + for node in nodes or []: + result.append(node) + result.extend(_flatten_lsblk(node.get("children"))) + return result + +def raid_status(): + mdstat, mdstat_error = "", "" + try: + mdstat = Path("/proc/mdstat").read_text() + except OSError as exc: + mdstat_error = str(exc) + array_name = Path(RAID_DEVICE).name + array_line = next((line for line in mdstat.splitlines() if line.startswith(f"{array_name} :")), "") + profile_match = re.search(r"(raid\d+)", array_line) + members_match = re.search(r"\[(\d+)/(\d+)\]\s+\[([^\]]+)\]", mdstat) + progress_match = re.search(r"(?:recovery|resync|reshape|check)\s*=\s*([\d.]+)%", mdstat, re.IGNORECASE) + finish_match = re.search(r"finish=([^\s]+)", mdstat) + speed_match = re.search(r"speed=([^\s]+)", mdstat) + mdadm_output, mdadm_error = _read_command(["mdadm", "--detail", RAID_DEVICE]) + detail = {} + for key, pattern in (("raid_devices", r"Raid Device\s*:\s*(\d+)"), ("active_devices", r"Active Devices\s*:\s*(\d+)"), + ("working_devices", r"Working Devices\s*:\s*(\d+)"), ("failed_devices", r"Failed Devices\s*:\s*(\d+)"), + ("spare_devices", r"Spare Devices\s*:\s*(\d+)")): + match = re.search(pattern, mdadm_output, re.IGNORECASE) + if match: + detail[key] = int(match.group(1)) + if profile_match: + detail["level"] = profile_match.group(1).upper() + elif mdadm_output: + level = re.search(r"Raid Level\s*:\s*(\S+)", mdadm_output, re.IGNORECASE) + detail["level"] = level.group(1).upper() if level else "RAID" + devices = [] + for line in mdadm_output.splitlines(): + match = re.match(r"\s*\d+\s+\d+\s+\d+\s+(\d+|-)\s+(.+?)\s+(/dev/\S+)\s*$", line) + if match: + state_text, device = match.group(2), match.group(3) + devices.append({"device": device, "slot": match.group(1), "state": state_text, "status": "active" if "active" in state_text.lower() else ("spare" if "spare" in state_text.lower() else "failed")}) + lsblk_output, _ = _read_command(["lsblk", "-J", "-b", "-o", "NAME,SIZE,MODEL,SERIAL,TYPE,PATH"]) + try: + block_devices = {item.get("path") or f"/dev/{item.get('name')}": item for item in _flatten_lsblk(json.loads(lsblk_output).get("blockdevices", []))} + except (ValueError, AttributeError): + block_devices = {} + for disk in devices: + info = block_devices.get(disk["device"], {}) + parent = re.sub(r"(p)?\d+$", "", disk["device"]) + info = info or block_devices.get(parent, {}) + if not info: + host_device = disk["device"].replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1) + host_lsblk, _ = _read_command(["lsblk", "-J", "-b", "-o", "NAME,SIZE,MODEL,SERIAL,TYPE,PATH", host_device]) + try: + host_items = _flatten_lsblk(json.loads(host_lsblk).get("blockdevices", [])) + info = host_items[0] if host_items else {} + except (ValueError, AttributeError): + info = {} + disk.update({"size": format_bytes(info.get("size") or 0) if info.get("size") else "—", "model": info.get("model") or "—", "serial": info.get("serial") or "—"}) + if SMART_ENABLED and shutil.which("smartctl"): + smart_device = disk["device"].replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1) + smart_output, _ = _read_command(["smartctl", "-H", smart_device], timeout=6) + health = re.search(r"SMART overall-health self-assessment test result:\s*(.+)", smart_output) + disk["smart"] = health.group(1).strip() if health else "Nicht verfügbar" + missing = members_match and members_match.group(1) != members_match.group(2) + failed = detail.get("failed_devices", 0) > 0 or any(disk["status"] == "failed" for disk in devices) + rebuilding = bool(progress_match) + if failed and not array_line: + status = "failed" + elif rebuilding: + status = "rebuilding" + elif failed or missing: + status = "degraded" + elif array_line or mdadm_output: + status = "ok" + else: + status = "failed" + return {"device": RAID_DEVICE, "status": status, "label": {"ok": "OK", "degraded": "DEGRADED", "failed": "FAILED", "rebuilding": "REBUILDING"}.get(status, "UNKNOWN"), + "level": detail.get("level", "—"), "raid_devices": detail.get("raid_devices") or (int(members_match.group(2)) if members_match else 0), + "active_devices": detail.get("active_devices") or (int(members_match.group(1)) if members_match else 0), "failed_devices": detail.get("failed_devices", 0), + "devices": devices, "recovery": {"active": rebuilding, "percent": float(progress_match.group(1)) if progress_match else 0, + "finish": finish_match.group(1) if finish_match else "—", "speed": speed_match.group(1) if speed_match else "—"}, + "error": None if array_line or mdadm_output else (mdadm_error or mdstat_error or "RAID-Status nicht verfügbar"), "updated_at": time.time()} + +def datastore_snapshot(): + now = time.time() + with datastore_cache_lock: + if datastore_cache["snapshot"] is None or now - datastore_cache["stored_at"] >= max(30, STORAGE_CACHE_SECONDS): + datastore_cache["snapshot"] = scan_datastore_storage() + datastore_cache["stored_at"] = now + storage = datastore_cache["snapshot"] + return {"storage": storage, "raid": raid_status(), "cache_seconds": STORAGE_CACHE_SECONDS} + def cpu_usage_percent(): global previous_cpu try: @@ -499,6 +663,14 @@ def system(): log.exception("Systemmetriken konnten nicht gelesen werden") return jsonify({"error": str(exc), "cpu_percent": 0, "memory": {}, "disks": []}), 500 +@app.get("/api/datastore") +def datastore(): + try: + return jsonify(datastore_snapshot()) + except Exception as exc: + log.exception("Datastore konnte nicht abgefragt werden") + return jsonify({"storage": {"error": str(exc), "path": DATASTORE_PATH, "categories": []}, "raid": {"status": "failed", "label": "FAILED", "error": str(exc)}}), 500 + @app.get("/health") def health(): return jsonify({"ok":True, "radarr_url":RADARR_URL, "sonarr_enabled":bool(SONARR_URL and SONARR_API_KEY), "sab_enabled":bool(SAB_URL and SAB_API_KEY)}) diff --git a/docker-compose.yml b/docker-compose.yml index 0e5fe97..aaef9ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: restart: unless-stopped ports: - "8099:8099" + devices: + - "${RAID_DEVICE:-/dev/md127}:${RAID_DEVICE:-/dev/md127}" extra_hosts: - "host.docker.internal:host-gateway" env_file: @@ -14,6 +16,10 @@ services: - ${FILM_MEDIA_VOLUME}:/media/filme:ro # Falls Sonarr verwendet wird, muss SERIES_MEDIA_VOLUME auf die Serien-Library zeigen. - ${SERIES_MEDIA_VOLUME:-/nesflix/serien}:/media/serien:ro + # Read-only Datastore für Größen-/Kategorie-Scans. + - ${DATASTORE_HOST_PATH:-/nesflix}:/nesflix:ro + # Read-only Host-Devices für lsblk/smartctl; Media Max führt keine RAID-Schreiboperationen aus. + - /dev:/host-dev:ro - ./data:/data networks: - media diff --git a/templates/index.html b/templates/index.html index cd354ce..5f387ea 100644 --- a/templates/index.html +++ b/templates/index.html @@ -30,12 +30,13 @@ {% for group in missing_german_series %}
{{ group.title }} ({{ group.year or '—' }}) · {{ group.episodes|length }} Episoden ohne Deutsch{% for r in group.episodes %}{% endfor %}
EpisodeQualitätStatusAudioUntertitelVideoGröße
{{ r.title }}{{ r.quality }}Ohne Deutsch
{% for l in r.audio_languages %}{{ l.flag }} {{ l.name }}{% else %}—{% endfor %}
{% for l in r.subtitle_languages %}{{ l.flag }} {{ l.name }}{% else %}—{% endfor %}
{{ r.video_codec }}{{ r.size }}
{% endfor %} {% if not missing_german_movies and not missing_german_series %}
Alle vorhandenen Filme und Episoden haben deutsches Audio oder deutsche Untertitel.
{% endif %} {% if sab_enabled %}
SABNZBD · DOWNLOAD CENTER

Download-Übersicht

Queue, Fortschritt und verbleibender Speicherplatz auf einen Blick.

Verbinde mit SABnzbd …
↓
Noch zu laden—Wird berechnet …
◉
Aktive Downloads—Verlauf: —
↯
Geschwindigkeit—Aktuelle Queue
✓
Status——
DownloadStatusFortschrittGröße / RestRestzeitQuelleSprache
Downloads werden geladen …
{% endif %} +
MEDIA MAX · DATASTORE

Datastore

Speicherbelegung, Medienkategorien und RAID-Status von /nesflix.

Wird geladen …
▣
Gesamtspeicher—
◒
Belegt—
◌
Frei—
%
Auslastung—

Gesamtauslastung

/nesflix

—

Speicher nach Kategorie

Anhand der tatsächlichen Verzeichnisse

Wird geladen …

RAID-Status

/dev/md127

Wird geladen …
———
Wird geladen …