Files
language-dashboard/app.py
T
nessi a90b516f49 fix: replace glob() with iterdir() in external_subtitle_files() to handle square brackets in media filenames
Replace glob pattern matching with iterdir() + startswith() filtering to avoid glob interpreting square brackets (e.g., [Bluray-1080p]) as pattern characters. Add OSError exception handling to return empty list when directory access fails.
2026-08-14 14:32:28 +02:00

409 lines
23 KiB
Python

import json
import logging
import os
import sqlite3
import subprocess
import threading
import time
import shutil
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from pathlib import Path
import requests
from flask import Flask, jsonify, render_template, request
RADARR_URL = os.environ.get("RADARR_URL", "http://radarr:7878").rstrip("/")
RADARR_API_KEY = os.environ.get("RADARR_API_KEY", "")
RADARR_MEDIA_PATH = os.environ.get("RADARR_MEDIA_PATH", "/data/filme").rstrip("/")
LOCAL_MEDIA_PATH = os.environ.get("LOCAL_MEDIA_PATH", "/media/filme").rstrip("/")
SONARR_URL = os.environ.get("SONARR_URL", "").rstrip("/")
SONARR_API_KEY = os.environ.get("SONARR_API_KEY", "")
SONARR_MEDIA_PATH = os.environ.get("SONARR_MEDIA_PATH", "/data/serien").rstrip("/")
LOCAL_SERIES_PATH = os.environ.get("LOCAL_SERIES_PATH", "/media/serien").rstrip("/")
SAB_URL = os.environ.get("SAB_URL", "").rstrip("/")
SAB_API_KEY = os.environ.get("SAB_API_KEY", "")
DB_PATH = os.environ.get("DB_PATH", "/data/cache.db")
REQUEST_TIMEOUT = int(os.environ.get("REQUEST_TIMEOUT", "20"))
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("radarr-language-dashboard")
app = Flask(__name__)
executor = ThreadPoolExecutor(max_workers=int(os.environ.get("SCAN_WORKERS", "2")))
scan_executor = ThreadPoolExecutor(max_workers=int(os.environ.get("SCAN_WORKERS", "4")))
jobs = {"radarr": {"state": "idle", "total": 0, "done": 0, "error": None}, "sonarr": {"state": "idle", "total": 0, "done": 0, "error": None}}
job_lock = threading.Lock()
system_lock = threading.Lock()
previous_cpu = None
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"}
def db():
Path(DB_PATH).parent.mkdir(parents=True, exist_ok=True)
con = sqlite3.connect(DB_PATH)
con.row_factory = sqlite3.Row
con.execute("CREATE TABLE IF NOT EXISTS media_cache (path TEXT PRIMARY KEY, mtime REAL NOT NULL, data TEXT NOT NULL, scanned_at TEXT NOT NULL)")
return con
def api_get(base, key, endpoint):
if not key:
raise RuntimeError(f"API-Key für {base} ist nicht gesetzt.")
r = requests.get(f"{base}/api/v3/{endpoint.lstrip('/')}", headers={"X-Api-Key": key}, timeout=REQUEST_TIMEOUT)
r.raise_for_status()
return r.json()
def sab_get(mode, **params):
if not SAB_URL or not SAB_API_KEY:
raise RuntimeError("SAB_URL oder SAB_API_KEY ist nicht gesetzt.")
query = {"output": "json", "apikey": SAB_API_KEY, "mode": mode, **params}
r = requests.get(f"{SAB_URL}/api", params=query, timeout=REQUEST_TIMEOUT)
r.raise_for_status()
data = r.json()
if data.get("error"):
raise RuntimeError(data["error"])
return data
def parse_size_bytes(value):
if isinstance(value, (int, float)):
return int(value)
match = re.search(r"([\d.,]+)\s*(B|KB|MB|GB|TB)", str(value or ""), re.IGNORECASE)
if not match:
return 0
number = float(match.group(1).replace(',', '.'))
multiplier = {"B": 1, "KB": 1024, "MB": 1024**2, "GB": 1024**3, "TB": 1024**4}[match.group(2).upper()]
return int(number * multiplier)
def format_bytes(size):
value = float(size or 0)
for unit in ("B", "KB", "MB", "GB", "TB"):
if value < 1024 or unit == "TB":
return f"{value:.1f} {unit}"
value /= 1024
def cpu_usage_percent():
global previous_cpu
try:
fields = Path("/proc/stat").read_text().splitlines()[0].split()[1:]
values = [int(value) for value in fields]
idle = values[3] + (values[4] if len(values) > 4 else 0)
total = sum(values)
with system_lock:
current = (total, idle)
if previous_cpu is None:
previous_cpu = current
return 0.0
total_delta = total - previous_cpu[0]
idle_delta = idle - previous_cpu[1]
previous_cpu = current
return round(max(0.0, min(100.0, (1 - idle_delta / total_delta) * 100)), 1) if total_delta else 0.0
except (OSError, ValueError, IndexError):
return 0.0
def system_metrics():
memory = {line.split(":", 1)[0]: int(line.split()[1]) * 1024 for line in Path("/proc/meminfo").read_text().splitlines() if ":" in line}
total_memory = memory.get("MemTotal", 0)
available_memory = memory.get("MemAvailable", memory.get("MemFree", 0))
used_memory = max(0, total_memory - available_memory)
# Filme und Serien liegen auf demselben Host-Speicher (/nesflix) und
# werden im Container über die Medien-Mounts sichtbar gemacht.
mount_paths = [("System", "/"), ("Medien", "/media/filme"), ("Medien", "/media/serien")]
disks, seen_devices = [], set()
for label, path in mount_paths:
try:
target = Path(path)
if not target.exists():
continue
device = target.stat().st_dev
if device in seen_devices:
continue
seen_devices.add(device)
usage = shutil.disk_usage(target)
disks.append({"label": label, "path": path, "total": format_bytes(usage.total), "used": format_bytes(usage.used), "free": format_bytes(usage.free), "used_bytes": usage.used, "total_bytes": usage.total, "percent": round(usage.used / usage.total * 100, 1) if usage.total else 0})
except OSError:
continue
return {"cpu_percent": cpu_usage_percent(), "memory": {"used": format_bytes(used_memory), "total": format_bytes(total_memory), "percent": round(used_memory / total_memory * 100, 1) if total_memory else 0}, "disks": disks, "timestamp": time.time()}
def release_languages(name):
value = f" {name.lower().replace('.', ' ').replace('-', ' ')} "
found = []
patterns = [("Deutsch", (" german ", " deutsch ", " german dubbed ", " ger dub ", " german dl ")), ("Englisch", (" english ", " eng ")), ("Japanisch", (" japanese ", " japan ", " jpn ")), ("Französisch", (" french ", " francais ", " fra ")), ("Spanisch", (" spanish ", " esp "))]
codes = {"Deutsch": "de", "Englisch": "en", "Japanisch": "ja", "Französisch": "fr", "Spanisch": "es"}
for language, tokens in patterns:
if any(token in value for token in tokens): found.append(norm_lang(codes[language]))
return unique_langs(found)
def sab_downloads():
queue = sab_get("queue", start=0, limit=100).get("queue") or {}
history = sab_get("history", start=0, limit=30, failed_only=0).get("history") or {}
wanted = []
try:
if RADARR_API_KEY: wanted.extend({"title": x.get("movie", {}).get("title") or x.get("title", ""), "source": "Radarr"} for x in api_get(RADARR_URL, RADARR_API_KEY, "queue?includeUnknownMovieItems=true").get("records", []))
if SONARR_URL and SONARR_API_KEY: wanted.extend({"title": x.get("title", ""), "source": "Sonarr"} for x in api_get(SONARR_URL, SONARR_API_KEY, "queue?includeUnknownSeriesItems=true").get("records", []))
except Exception:
log.warning("Radarr/Sonarr-Queue konnte für SAB-Zuordnung nicht geladen werden", exc_info=True)
def make_item(item, completed=False):
title = item.get("name") or item.get("filename") or item.get("nzb_name") or "—"
title_lower = title.lower()
source = "SABnzbd"
for candidate in wanted:
candidate_title = candidate["title"].lower()
if candidate_title and (candidate_title in title_lower or title_lower in candidate_title): source = candidate["source"]; break
percentage = 100.0 if completed else float(item.get("percentage") or 0)
return {"title": title, "status": item.get("status") or ("Completed" if completed else "Queued"), "percentage": percentage, "size": item.get("size") or "—", "remaining": item.get("sizeleft") or ("0 B" if completed else "—"), "timeleft": item.get("timeleft") or "—", "speed": item.get("speed") or "—", "category": item.get("cat") or item.get("category") or "—", "source": source, "languages": release_languages(title), "completed": completed}
items = [make_item(item) for item in queue.get("slots", [])]
items.extend(make_item(item, completed=True) for item in history.get("slots", [])[:15])
remaining_bytes = sum(parse_size_bytes(item.get("sizeleft")) for item in queue.get("slots", []))
return {"items": items, "queue_status": queue.get("status", "Idle"), "speed": queue.get("speed", "0 B/s"),
"paused": bool(queue.get("paused_all")), "active_count": len(queue.get("slots", [])),
"history_count": len(history.get("slots", [])[:15]), "remaining_bytes": remaining_bytes,
"remaining": format_bytes(remaining_bytes), "error": None}
def norm_lang(code):
code = (code or "").strip().lower()
name = LANG_NAMES.get(code, code.upper() if code else "Unbekannt")
return {"code": code or "und", "name": name, "flag": FLAGS.get(name, "🌐")}
def unique_langs(items):
out, seen = [], set()
for item in items:
if item["name"] not in seen:
seen.add(item["name"]); out.append(item)
return out
def map_path(path, remote, local):
if not path: return None
if remote and path.startswith(remote):
return str(Path(local) / path[len(remote):].lstrip("/"))
return path
EXTERNAL_SUBTITLE_EXTENSIONS = {".srt", ".ass", ".ssa", ".vtt", ".sub", ".idx", ".sup"}
def external_subtitle_files(path):
media = Path(path)
prefix = f"{media.stem}."
# Nicht glob() verwenden: Mediennamen enthalten oft eckige Klammern
# wie [Bluray-1080p], die glob als Pattern-Zeichen interpretieren würde.
try:
return sorted(candidate for candidate in media.parent.iterdir() if candidate.is_file() and candidate.name.startswith(prefix) and candidate.suffix.lower() in EXTERNAL_SUBTITLE_EXTENSIONS)
except OSError:
return []
def external_subtitle_languages(path):
media = Path(path)
languages = []
for candidate in external_subtitle_files(path):
tokens = {token for token in re.split(r"[. _()\[\]-]+", candidate.stem.lower()) if token}
if tokens & {"de", "deu", "ger", "german", "deutsch"}:
languages.append(norm_lang("de"))
if tokens & {"en", "eng", "english"}:
languages.append(norm_lang("en"))
if tokens & {"fr", "fra", "fre", "french", "französisch"}:
languages.append(norm_lang("fr"))
if tokens & {"es", "spa", "spanish", "spanisch"}:
languages.append(norm_lang("es"))
if tokens & {"it", "ita", "italian", "italienisch"}:
languages.append(norm_lang("it"))
return unique_langs(languages)
def run_ffprobe(path):
proc = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=size:stream=index,codec_type,codec_name,channels,channel_layout:stream_tags=language,title", "-of", "json", path], capture_output=True, text=True, timeout=90)
if proc.returncode != 0: raise RuntimeError(proc.stderr.strip() or "ffprobe fehlgeschlagen")
raw, audios, subs, details, video = json.loads(proc.stdout or "{}"), [], [], [], None
for stream in raw.get("streams", []):
tags = stream.get("tags") or {}; lang = norm_lang(tags.get("language")); kind = stream.get("codec_type")
if kind == "video" and not video: video = stream.get("codec_name")
elif kind == "audio":
audios.append(lang); details.append({"language": lang, "codec": stream.get("codec_name"), "channels": stream.get("channels"), "layout": stream.get("channel_layout"), "title": tags.get("title")})
elif kind == "subtitle": subs.append(lang)
subtitles = unique_langs(subs + external_subtitle_languages(path))
return {"audio_languages": unique_langs(audios), "subtitle_languages": subtitles, "external_subtitle_files": [str(file) for file in external_subtitle_files(path)], "audio_details": details, "video_codec": video or "—", "size": int((raw.get("format") or {}).get("size") or 0)}
def cached(path):
p = Path(path)
if not p.exists(): return {"error": f"Datei nicht gefunden: {p}", "pending": True}
mtime = p.stat().st_mtime; con = db(); row = con.execute("SELECT mtime,data FROM media_cache WHERE path=?", (str(p),)).fetchone(); con.close()
if row and float(row["mtime"]) == float(mtime):
data = json.loads(row["data"])
current_subtitles = [str(file) for file in external_subtitle_files(path)]
if "external_subtitle_files" in data and data["external_subtitle_files"] == current_subtitles:
return data
return None
def scan_one(path):
p = Path(path)
if not p.exists(): data = {"error": f"Datei nicht gefunden: {p}"}
else:
try: data = run_ffprobe(str(p)); log.info("Gescannt: %s", p)
except Exception as exc: data = {"error": str(exc)}; log.exception("Scanfehler für %s", p)
mtime = p.stat().st_mtime
con = db(); con.execute("INSERT INTO media_cache(path,mtime,data,scanned_at) VALUES(?,?,?,?) ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime,data=excluded.data,scanned_at=excluded.scanned_at", (str(p), mtime, json.dumps(data), datetime.utcnow().isoformat())); con.commit(); con.close()
return data
def format_size(size):
if not size: return "—"
value = float(size)
for unit in ["B", "KB", "MB", "GB", "TB"]:
if value < 1024 or unit == "TB": return f"{value:.1f} {unit}"
value /= 1024
def language_state(audios):
names = {x["name"] for x in audios}
if {"Deutsch", "Englisch"} <= names: return {"class":"ok", "label":"DE + EN"}
if "Deutsch" in names: return {"class":"warn", "label":"Deutsch"}
if "Englisch" in names: return {"class":"bad", "label":"English only"}
return {"class":"neutral", "label":"Andere"}
def has_german_track(row):
languages = row.get("audio_languages", []) + row.get("subtitle_languages", [])
return any(language.get("name") == "Deutsch" for language in languages)
def is_german_review_candidate(row):
"""Only review media that exists or is currently being downloaded."""
if row.get("episode") and int(row.get("season") or 0) == 0:
return False
return not row.get("missing") or bool(row.get("downloading"))
def media_row(name, year, media_file, remote, local, extra=None):
path = media_file.get("path") if media_file else None
if not path and media_file and media_file.get("relativePath"): path = str(Path(media_file.get("basePath") or "") / media_file["relativePath"])
local_path = map_path(path, remote, local); info = cached(local_path) if local_path else {"error":"Kein Dateipfad erhalten"}
info = info or {"pending": True}; audios = info.get("audio_languages", [])
if local_path:
info["subtitle_languages"] = unique_langs(info.get("subtitle_languages", []) + external_subtitle_languages(local_path))
downloading = bool((extra or {}).get("downloading"))
quality = ((((media_file or {}).get("quality") or {}).get("quality") or {}).get("name")) or "—"
row = {"title":name, "year":year, "quality":"Downloading" if downloading else (quality if media_file else "Keine Datei"), "audio_languages":audios, "subtitle_languages":info.get("subtitle_languages", []), "video_codec":info.get("video_codec", "—"), "size":format_size(info.get("size") or (media_file or {}).get("size")), "radarr_path":path or "—", "local_path":local_path or "—", "state":{"class":"downloading", "label":"Downloading"} if downloading else (language_state(audios) if audios else {"class":"neutral", "label":"Scan ausstehend" if info.get("pending") else ("Fehler" if info.get("error") else "Keine Spuren")}), "error":info.get("error"), "missing":not bool(media_file), "pending":bool(info.get("pending")), "downloading":downloading}
if extra: row.update(extra)
return row
def radarr_rows():
rows = []
queue = api_get(RADARR_URL, RADARR_API_KEY, "queue?includeUnknownMovieItems=true")
queued_movies = {item.get("movieId") for item in queue.get("records", [])}
for movie in api_get(RADARR_URL, RADARR_API_KEY, "movie"):
mf = movie.get("movieFile"); path = (mf or {}).get("path") or ((movie.get("path") and (mf or {}).get("relativePath")) and str(Path(movie["path"]) / mf["relativePath"]))
if mf and path: mf = dict(mf); mf["path"] = path
rows.append(media_row(movie.get("title", "—"), movie.get("year"), mf, RADARR_MEDIA_PATH, LOCAL_MEDIA_PATH, {"downloading": movie.get("id") in queued_movies}))
return sorted(rows, key=lambda x: x["title"].lower())
def sonarr_groups():
groups = []
queue = api_get(SONARR_URL, SONARR_API_KEY, "queue?includeUnknownSeriesItems=true")
queued_episodes = {item.get("episodeId") for item in queue.get("records", [])}
for series in api_get(SONARR_URL, SONARR_API_KEY, "series"):
episodes = api_get(SONARR_URL, SONARR_API_KEY, f"episode?seriesId={series['id']}&includeSeries=false")
files = {f["id"]: f for f in api_get(SONARR_URL, SONARR_API_KEY, f"episodefile?seriesId={series['id']}")}
items = []
for episode in sorted(episodes, key=lambda x: (x.get("seasonNumber", 0), x.get("episodeNumber", 0))):
ef = files.get(episode.get("episodeFileId"));
if ef: ef = dict(ef); ef["path"] = str(Path(series.get("path") or "") / ef.get("relativePath", ""))
label = f"S{episode.get('seasonNumber', 0):02d}E{episode.get('episodeNumber', 0):02d} · {episode.get('title') or '—'}"
items.append(media_row(label, None, ef, SONARR_MEDIA_PATH, LOCAL_SERIES_PATH, {"episode":True, "season":episode.get("seasonNumber", 0), "downloading": episode.get("id") in queued_episodes}))
groups.append({"title":series.get("title", "—"), "year":series.get("year"), "episodes":items})
return sorted(groups, key=lambda x: x["title"].lower())
def start_scan(source, force=False):
with job_lock:
if jobs[source]["state"] == "running" or (jobs[source]["state"] in ("done", "error") and not force): return
jobs[source] = {"state":"running", "total":0, "done":0, "error":None}
def work():
try:
paths = []
if source == "radarr":
for row in radarr_rows():
if row["local_path"] != "—" and row["pending"]: paths.append(row["local_path"])
else:
for group in sonarr_groups():
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and r["pending"])
with job_lock: jobs[source]["total"] = len(paths)
futures = [scan_executor.submit(scan_one, path) for path in paths]
for future in as_completed(futures):
future.result()
with job_lock: jobs[source]["done"] += 1
with job_lock: jobs[source]["state"] = "done"
except Exception as exc:
log.exception("%s-Scan fehlgeschlagen", source); jobs[source]["error"] = str(exc); jobs[source]["state"] = "error"
executor.submit(work)
@app.route("/")
def index():
error = None; rows = []; series = []
try:
rows = radarr_rows()
if any(row["pending"] for row in rows): start_scan("radarr")
except Exception as exc: error = str(exc)
if SONARR_URL and SONARR_API_KEY:
try:
series = sonarr_groups()
if any(row["pending"] for group in series for row in group["episodes"]): start_scan("sonarr")
except Exception as exc: log.exception("Sonarr nicht erreichbar"); error = f"{error + ' | ' if error else ''}Sonarr: {exc}"
missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not has_german_track(row)]
missing_german_series = []
for group in series:
episodes = [episode for episode in group["episodes"] if is_german_review_candidate(episode) and not has_german_track(episode)]
if episodes:
missing_german_series.append({**group, "episodes": episodes})
dashboard_stats = {
"movie_total": len(rows),
"movie_files": sum(1 for row in rows if not row.get("missing")),
"movie_missing": sum(1 for row in rows if row.get("missing") and not row.get("downloading")),
"movie_downloading": sum(1 for row in rows if row.get("downloading")),
"series_total": len(series),
"episode_total": sum(len(group["episodes"]) for group in series),
"episode_files": sum(1 for group in series for row in group["episodes"] if not row.get("missing")),
"episode_missing": sum(1 for group in series for row in group["episodes"] if row.get("missing") and not row.get("downloading")),
}
return render_template("index.html", rows=rows, series=series, missing_german_movies=missing_german_movies,
missing_german_series=missing_german_series, error=error,
dashboard_stats=dashboard_stats,
sonarr_enabled=bool(SONARR_URL and SONARR_API_KEY), sab_enabled=bool(SAB_URL and SAB_API_KEY),
radarr_media_path=RADARR_MEDIA_PATH, local_media_path=LOCAL_MEDIA_PATH,
local_series_path=LOCAL_SERIES_PATH)
@app.post("/api/rescan")
def api_rescan():
source = request.args.get("source", "radarr")
if source not in (*jobs.keys(), "all"): return jsonify({"ok":False, "error":"Unbekannte Quelle"}), 400
con = db()
if source in ("radarr", "all"): con.execute("DELETE FROM media_cache WHERE path NOT LIKE ?", (LOCAL_SERIES_PATH.rstrip("/") + "/%",))
if source in ("sonarr", "all"): con.execute("DELETE FROM media_cache WHERE path LIKE ?", (LOCAL_SERIES_PATH.rstrip("/") + "/%",))
con.commit(); con.close()
if source == "all":
start_scan("radarr", force=True)
if SONARR_URL and SONARR_API_KEY:
start_scan("sonarr", force=True)
else:
start_scan(source, force=True)
return jsonify({"ok":True})
@app.get("/api/scan-status")
def scan_status():
with job_lock: return jsonify(jobs)
@app.get("/api/downloads")
def downloads():
if not SAB_URL or not SAB_API_KEY:
return jsonify({"items": [], "queue_status": "Nicht konfiguriert", "speed": "—", "paused": False, "active_count": 0, "history_count": 0, "remaining": "—", "error": "SAB_URL oder SAB_API_KEY ist nicht gesetzt."})
try:
return jsonify(sab_downloads())
except Exception as exc:
log.exception("SABnzbd konnte nicht abgefragt werden")
return jsonify({"items": [], "queue_status": "Fehler", "speed": "—", "paused": False, "active_count": 0, "history_count": 0, "remaining": "—", "error": str(exc)}), 502
@app.get("/api/system")
def system():
try:
return jsonify(system_metrics())
except Exception as exc:
log.exception("Systemmetriken konnten nicht gelesen werden")
return jsonify({"error": str(exc), "cpu_percent": 0, "memory": {}, "disks": []}), 500
@app.get("/health")
def health(): return jsonify({"ok":True, "radarr_url":RADARR_URL, "sonarr_enabled":bool(SONARR_URL and SONARR_API_KEY), "sab_enabled":bool(SAB_URL and SAB_API_KEY)})
if __name__ == "__main__": app.run(host="0.0.0.0", port=8099)