feat: add datastore view with storage categories, RAID status monitoring, and live recovery tracking

Add scan_datastore_storage() to walk DATASTORE_PATH and categorize media by directory structure (movies/series/anime/kdrama/downloads/other) with size/file counts. Add raid_status() to parse /proc/mdstat and mdadm --detail for RAID level, active/failed devices, and recovery progress. Add /api/datastore endpoint with 15-minute cache for storage scans and live RAID updates. Install mdadm/util-linux/

app.py

@@ -38,6 +38,11 @@ SERVICE_CHECK_TIMEOUT = float(os.environ.get("SERVICE_CHECK_TIMEOUT", "3"))
 SERVICE_REFRESH_SECONDS = int(os.environ.get("SERVICE_REFRESH_SECONDS", "30"))
 DB_PATH = os.environ.get("DB_PATH", "/data/cache.db")
 REQUEST_TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT", "20"))
+DATASTORE_PATH = os.environ.get("DATASTORE_PATH", "/nesflix").rstrip("/") or "/"
+RAID_DEVICE = os.environ.get("RAID_DEVICE", "/dev/md127")
+HOST_DEVICE_PATH = os.environ.get("HOST_DEVICE_PATH", "/host-dev")
+STORAGE_CACHE_SECONDS = int(os.environ.get("STORAGE_CACHE_SECONDS", "900"))
+SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"}
 
 logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
 log = logging.getLogger("media-max")
@@ -50,6 +55,8 @@ system_lock = threading.Lock()
 previous_cpu = None
 service_status_lock = threading.Lock()
 service_check_executor = ThreadPoolExecutor(max_workers=16)
+datastore_cache_lock = threading.Lock()
+datastore_cache = {"stored_at": 0.0, "snapshot": None}
 
 LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
 FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"}
@@ -160,6 +167,163 @@ def format_bytes(size):
             return f"{value:.1f} {unit}"
         value /= 1024
 
+def _normalized_path_part(value):
+    return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold())
+
+def _datastore_category(relative_path, category_roots):
+    parts = [part for part in Path(relative_path).parts if part not in (".", "")]
+    normalized = {_normalized_path_part(part) for part in parts}
+    if normalized & {"download", "downloads", "sabnzbd", "nzb"}:
+        return "downloads"
+    if normalized & {"anime"}:
+        return "anime"
+    if normalized & {"kdrama", "kdramas", "kdramen", "korean", "koreanischeserien"}:
+        return "kdrama"
+    if parts and _normalized_path_part(parts[0]) in category_roots["movies"]:
+        return "movies"
+    if parts and _normalized_path_part(parts[0]) in category_roots["series"]:
+        return "series"
+    return "other"
+
+def scan_datastore_storage():
+    root = Path(DATASTORE_PATH)
+    categories = {key: {"key": key, "label": label, "bytes": 0, "files": 0} for key, label in (
+        ("movies", "Filme"), ("series", "Serien"), ("anime", "Anime"),
+        ("kdrama", "K-Dramen"), ("downloads", "Downloads"), ("other", "Sonstiges")
+    )}
+    if not root.exists() or not root.is_dir():
+        return {"error": f"Datastore {DATASTORE_PATH} ist nicht verfügbar.", "path": DATASTORE_PATH, "categories": list(categories.values())}
+    category_roots = {
+        "movies": {"filme", "film", "movies", "movie"},
+        "series": {"serien", "serie", "series", "tv", "shows"},
+    }
+    try:
+        for current, directories, files in os.walk(root, followlinks=False):
+            directories[:] = [directory for directory in directories if not os.path.islink(os.path.join(current, directory))]
+            relative = os.path.relpath(current, root)
+            for filename in files:
+                full_path = os.path.join(current, filename)
+                if os.path.islink(full_path):
+                    continue
+                try:
+                    size = os.path.getsize(full_path)
+                except OSError:
+                    continue
+                category = categories[_datastore_category(relative, category_roots)]
+                category["bytes"] += size
+                category["files"] += 1
+        usage = shutil.disk_usage(root)
+    except OSError as exc:
+        return {"error": f"Datastore konnte nicht gelesen werden: {exc}", "path": DATASTORE_PATH, "categories": list(categories.values())}
+    used = usage.used
+    values = []
+    for category in categories.values():
+        category["size"] = format_bytes(category["bytes"])
+        category["percent"] = round(category["bytes"] / used * 100, 1) if used else 0
+        values.append(category)
+    return {"error": None, "path": DATASTORE_PATH, "total_bytes": usage.total, "used_bytes": usage.used,
+            "free_bytes": usage.free, "total": format_bytes(usage.total), "used": format_bytes(usage.used),
+            "free": format_bytes(usage.free), "percent": round(usage.used / usage.total * 100, 1) if usage.total else 0,
+            "categories": values, "scanned_at": time.time()}
+
+def _read_command(command, timeout=4):
+    try:
+        result = subprocess.run(command, capture_output=True, text=True, timeout=timeout, check=False)
+        return result.stdout, result.stderr
+    except (OSError, subprocess.SubprocessError) as exc:
+        return "", str(exc)
+
+def _flatten_lsblk(nodes):
+    result = []
+    for node in nodes or []:
+        result.append(node)
+        result.extend(_flatten_lsblk(node.get("children")))
+    return result
+
+def raid_status():
+    mdstat, mdstat_error = "", ""
+    try:
+        mdstat = Path("/proc/mdstat").read_text()
+    except OSError as exc:
+        mdstat_error = str(exc)
+    array_name = Path(RAID_DEVICE).name
+    array_line = next((line for line in mdstat.splitlines() if line.startswith(f"{array_name} :")), "")
+    profile_match = re.search(r"(raid\d+)", array_line)
+    members_match = re.search(r"\[(\d+)/(\d+)\]\s+\[([^\]]+)\]", mdstat)
+    progress_match = re.search(r"(?:recovery|resync|reshape|check)\s*=\s*([\d.]+)%", mdstat, re.IGNORECASE)
+    finish_match = re.search(r"finish=([^\s]+)", mdstat)
+    speed_match = re.search(r"speed=([^\s]+)", mdstat)
+    mdadm_output, mdadm_error = _read_command(["mdadm", "--detail", RAID_DEVICE])
+    detail = {}
+    for key, pattern in (("raid_devices", r"Raid Device\s*:\s*(\d+)"), ("active_devices", r"Active Devices\s*:\s*(\d+)"),
+                         ("working_devices", r"Working Devices\s*:\s*(\d+)"), ("failed_devices", r"Failed Devices\s*:\s*(\d+)"),
+                         ("spare_devices", r"Spare Devices\s*:\s*(\d+)")):
+        match = re.search(pattern, mdadm_output, re.IGNORECASE)
+        if match:
+            detail[key] = int(match.group(1))
+    if profile_match:
+        detail["level"] = profile_match.group(1).upper()
+    elif mdadm_output:
+        level = re.search(r"Raid Level\s*:\s*(\S+)", mdadm_output, re.IGNORECASE)
+        detail["level"] = level.group(1).upper() if level else "RAID"
+    devices = []
+    for line in mdadm_output.splitlines():
+        match = re.match(r"\s*\d+\s+\d+\s+\d+\s+(\d+|-)\s+(.+?)\s+(/dev/\S+)\s*$", line)
+        if match:
+            state_text, device = match.group(2), match.group(3)
+            devices.append({"device": device, "slot": match.group(1), "state": state_text, "status": "active" if "active" in state_text.lower() else ("spare" if "spare" in state_text.lower() else "failed")})
+    lsblk_output, _ = _read_command(["lsblk", "-J", "-b", "-o", "NAME,SIZE,MODEL,SERIAL,TYPE,PATH"])
+    try:
+        block_devices = {item.get("path") or f"/dev/{item.get('name')}": item for item in _flatten_lsblk(json.loads(lsblk_output).get("blockdevices", []))}
+    except (ValueError, AttributeError):
+        block_devices = {}
+    for disk in devices:
+        info = block_devices.get(disk["device"], {})
+        parent = re.sub(r"(p)?\d+$", "", disk["device"])
+        info = info or block_devices.get(parent, {})
+        if not info:
+            host_device = disk["device"].replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1)
+            host_lsblk, _ = _read_command(["lsblk", "-J", "-b", "-o", "NAME,SIZE,MODEL,SERIAL,TYPE,PATH", host_device])
+            try:
+                host_items = _flatten_lsblk(json.loads(host_lsblk).get("blockdevices", []))
+                info = host_items[0] if host_items else {}
+            except (ValueError, AttributeError):
+                info = {}
+        disk.update({"size": format_bytes(info.get("size") or 0) if info.get("size") else "—", "model": info.get("model") or "—", "serial": info.get("serial") or "—"})
+        if SMART_ENABLED and shutil.which("smartctl"):
+            smart_device = disk["device"].replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1)
+            smart_output, _ = _read_command(["smartctl", "-H", smart_device], timeout=6)
+            health = re.search(r"SMART overall-health self-assessment test result:\s*(.+)", smart_output)
+            disk["smart"] = health.group(1).strip() if health else "Nicht verfügbar"
+    missing = members_match and members_match.group(1) != members_match.group(2)
+    failed = detail.get("failed_devices", 0) > 0 or any(disk["status"] == "failed" for disk in devices)
+    rebuilding = bool(progress_match)
+    if failed and not array_line:
+        status = "failed"
+    elif rebuilding:
+        status = "rebuilding"
+    elif failed or missing:
+        status = "degraded"
+    elif array_line or mdadm_output:
+        status = "ok"
+    else:
+        status = "failed"
+    return {"device": RAID_DEVICE, "status": status, "label": {"ok": "OK", "degraded": "DEGRADED", "failed": "FAILED", "rebuilding": "REBUILDING"}.get(status, "UNKNOWN"),
+            "level": detail.get("level", "—"), "raid_devices": detail.get("raid_devices") or (int(members_match.group(2)) if members_match else 0),
+            "active_devices": detail.get("active_devices") or (int(members_match.group(1)) if members_match else 0), "failed_devices": detail.get("failed_devices", 0),
+            "devices": devices, "recovery": {"active": rebuilding, "percent": float(progress_match.group(1)) if progress_match else 0,
+            "finish": finish_match.group(1) if finish_match else "—", "speed": speed_match.group(1) if speed_match else "—"},
+            "error": None if array_line or mdadm_output else (mdadm_error or mdstat_error or "RAID-Status nicht verfügbar"), "updated_at": time.time()}
+
+def datastore_snapshot():
+    now = time.time()
+    with datastore_cache_lock:
+        if datastore_cache["snapshot"] is None or now - datastore_cache["stored_at"] >= max(30, STORAGE_CACHE_SECONDS):
+            datastore_cache["snapshot"] = scan_datastore_storage()
+            datastore_cache["stored_at"] = now
+        storage = datastore_cache["snapshot"]
+    return {"storage": storage, "raid": raid_status(), "cache_seconds": STORAGE_CACHE_SECONDS}
+
 def cpu_usage_percent():
     global previous_cpu
     try:
@@ -499,6 +663,14 @@ def system():
         log.exception("Systemmetriken konnten nicht gelesen werden")
         return jsonify({"error": str(exc), "cpu_percent": 0, "memory": {}, "disks": []}), 500
 
+@app.get("/api/datastore")
+def datastore():
+    try:
+        return jsonify(datastore_snapshot())
+    except Exception as exc:
+        log.exception("Datastore konnte nicht abgefragt werden")
+        return jsonify({"storage": {"error": str(exc), "path": DATASTORE_PATH, "categories": []}, "raid": {"status": "failed", "label": "FAILED", "error": str(exc)}}), 500
+
 @app.get("/health")
 def health(): return jsonify({"ok":True, "radarr_url":RADARR_URL, "sonarr_enabled":bool(SONARR_URL and SONARR_API_KEY), "sab_enabled":bool(SAB_URL and SAB_API_KEY)})