nessi/language-dashboard
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: redesign dashboard with Tracearr-inspired layout and improve language detection

Browse Source
This commit is contained in:
nessi committed 2026-09-30 16:07:54 +02:00
1 parent 00b841fa67
commit 15310c5e61
8 files changed
+895 -314
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app.py
+119 -57
View File
@@ -9,7 +9,7 @@ import time
import shutil
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime
from datetime import datetime, timezone
from pathlib import Path
import requests
@@ -99,8 +99,42 @@ hardware_log_once = set()
library_cache_lock = threading.Lock()
library_cache = {"rows": [], "series": [], "updated_at": 0.0, "error": None, "ready": False}
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"}
LANGUAGE_DEFINITIONS = [
("de", "Deutsch", "🇩🇪", "deu ger german deutsch"),
("en", "Englisch", "🇬🇧", "eng english englisch"),
("ja", "Japanisch", "🇯🇵", "jpn japanese japanisch"),
("ko", "Koreanisch", "🇰🇷", "kor korean koreanisch"),
("fr", "Französisch", "🇫🇷", "fra fre french français francais französisch"),
("es", "Spanisch", "🇪🇸", "spa spanish spanisch español espanol"),
("it", "Italienisch", "🇮🇹", "ita italian italienisch"),
("ru", "Russisch", "🇷🇺", "rus russian russisch"),
("zh", "Chinesisch", "🇨🇳", "zho chi chinese chinesisch mandarin cmn cantonese yue"),
("pt", "Portugiesisch", "🇵🇹", "por portuguese portugiesisch português portugues"),
("nl", "Niederländisch", "🇳🇱", "nld dut dutch niederländisch"),
("pl", "Polnisch", "🇵🇱", "pol polish polnisch"),
("tr", "Türkisch", "🇹🇷", "tur turkish türkisch"),
("ar", "Arabisch", "🇸🇦", "ara arabic arabisch"),
("hi", "Hindi", "🇮🇳", "hin hindi"),
("sv", "Schwedisch", "🇸🇪", "swe swedish schwedisch"),
("da", "Dänisch", "🇩🇰", "dan danish dänisch"),
("no", "Norwegisch", "🇳🇴", "nor nob nno nb nn norwegian norwegisch"),
("fi", "Finnisch", "🇫🇮", "fin finnish finnisch"),
("cs", "Tschechisch", "🇨🇿", "ces cze czech tschechisch"),
("el", "Griechisch", "🇬🇷", "ell gre greek griechisch"),
("hu", "Ungarisch", "🇭🇺", "hun hungarian ungarisch"),
("uk", "Ukrainisch", "🇺🇦", "ukr ukrainian ukrainisch"),
("ro", "Rumänisch", "🇷🇴", "ron rum romanian rumänisch"),
("th", "Thai", "🇹🇭", "tha thai"),
("vi", "Vietnamesisch", "🇻🇳", "vie vietnamese vietnamesisch"),
("id", "Indonesisch", "🇮🇩", "ind indonesian indonesisch"),
("he", "Hebräisch", "🇮🇱", "heb hebrew hebräisch"),
]
LANG_ALIASES = {alias: code for code, name, flag, aliases in LANGUAGE_DEFINITIONS
for alias in (code, name.casefold(), *aliases.split())}
LANG_NAMES = {code: name for code, name, flag, aliases in LANGUAGE_DEFINITIONS}
LANG_NAMES["und"] = "Unbekannt"
FLAGS = {name: flag for code, name, flag, aliases in LANGUAGE_DEFINITIONS}
FLAGS["Unbekannt"] = "❓"
SERVICE_DEFINITIONS = [
{"key": "radarr", "name": "Radarr", "url": f"{RADARR_URL}/api/v3/system/status", "api_key": RADARR_API_KEY, "kind": "api"},
@@ -767,7 +801,7 @@ def system_metrics():
return {"cpu_percent": cpu_usage_percent(), "memory": {"used": format_bytes(used_memory), "total": format_bytes(total_memory), "percent": round(used_memory / total_memory * 100, 1) if total_memory else 0}, "disks": disks, "load": [round(value, 2) for value in load], "uptime_seconds": round(uptime_seconds), "processes": process_count, "timestamp": time.time()}
def release_languages(name):
value = f" {name.lower().replace('.', ' ').replace('-', ' ')} "
value = " " + re.sub(r"[\W_]+", " ", str(name or "").casefold()).strip() + " "
found = []
patterns = [
("Deutsch", (" german ", " german dubbed ", " deutsch ", " ger ", " deu ", " german dl ")),
@@ -816,9 +850,12 @@ def sab_downloads():
"remaining": format_bytes(remaining_bytes), "error": None}
def norm_lang(code):
code = (code or "").strip().lower()
name = LANG_NAMES.get(code, code.upper() if code else "Unbekannt")
return {"code": code or "und", "name": name, "flag": FLAGS.get(name, "🌐")}
value = str(code or "").strip().casefold()
code = LANG_ALIASES.get(value) or LANG_ALIASES.get(re.split(r"[-_]", value)[0]) or value
if code in {"", "und", "unknown", "undefined", "unbekannt", "unk", "mul", "zxx"}:
code = "und"
name = LANG_NAMES.get(code, code.upper())
return {"code": code, "name": name, "flag": FLAGS.get(name, "🌐")}
def unique_langs(items):
out, seen = [], set()
@@ -829,12 +866,13 @@ def unique_langs(items):
def map_path(path, remote, local):
if not path: return None
if remote and path.startswith(remote):
if remote and (path == remote or path.startswith(remote.rstrip("/") + "/")):
return str(Path(local) / path[len(remote):].lstrip("/"))
return path
EXTERNAL_SUBTITLE_EXTENSIONS = {".srt", ".ass", ".ssa", ".vtt", ".sub", ".idx", ".sup"}
SCANNER_VERSION = 2
SCANNER_VERSION = 3
SCAN_ERROR_RETRY_SECONDS = 300
def external_subtitle_files(path):
media = Path(path)
@@ -850,60 +888,51 @@ def external_subtitle_languages(path):
media = Path(path)
languages = []
for candidate in external_subtitle_files(path):
tokens = {token for token in re.split(r"[. _()\[\]-]+", candidate.stem.lower()) if token}
if tokens & {"de", "deu", "ger", "german", "deutsch"}:
languages.append(norm_lang("de"))
if tokens & {"en", "eng", "english"}:
languages.append(norm_lang("en"))
if tokens & {"fr", "fra", "fre", "french", "französisch"}:
languages.append(norm_lang("fr"))
if tokens & {"es", "spa", "spanish", "spanisch"}:
languages.append(norm_lang("es"))
if tokens & {"it", "ita", "italian", "italienisch"}:
languages.append(norm_lang("it"))
# Only inspect the suffix, never words from the movie title itself.
suffix = candidate.stem[len(media.stem):]
languages.extend(languages_from_text(suffix))
return unique_langs(languages)
def languages_from_text(value):
tokens = re.findall(r"[^\W_]+", str(value or "").casefold())
return unique_langs([norm_lang(LANG_ALIASES[token]) for token in tokens if token in LANG_ALIASES])
def language_from_text(value):
"""Infer a language from a subtitle/audio track title when the codec tag is `und`."""
tokens = {token for token in re.split(r"[. _()\[\]-]+", str(value or "").lower()) if token}
aliases = {
"de": "de", "deu": "de", "ger": "de", "german": "de", "deutsch": "de",
"en": "en", "eng": "en", "english": "en",
"ja": "ja", "jpn": "ja", "japanese": "ja", "japanisch": "ja",
"ko": "ko", "kor": "ko", "korean": "ko", "koreanisch": "ko",
"fr": "fr", "fra": "fr", "fre": "fr", "french": "fr", "französisch": "fr",
"es": "es", "spa": "es", "spanish": "es", "spanisch": "es",
"it": "it", "ita": "it", "italian": "it", "italienisch": "it",
}
for token in tokens:
if token in aliases:
return norm_lang(aliases[token])
return None
"""Use unambiguous track titles; do not guess between multiple languages."""
languages = languages_from_text(value)
return languages[0] if len(languages) == 1 else None
def run_ffprobe(path):
proc = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=size:stream=index,codec_type,codec_name,channels,channel_layout:stream_tags=language,title", "-of", "json", path], capture_output=True, text=True, timeout=90)
if proc.returncode != 0: raise RuntimeError(proc.stderr.strip() or "ffprobe fehlgeschlagen")
raw, audios, subs, details, video = json.loads(proc.stdout or "{}"), [], [], [], None
for stream in raw.get("streams", []):
tags = stream.get("tags") or {}; kind = stream.get("codec_type")
tags = {key.casefold(): value for key, value in (stream.get("tags") or {}).items()}; kind = stream.get("codec_type")
lang = norm_lang(tags.get("language"))
# Viele ASS-Spuren werden von ffprobe nur als `und` gemeldet, obwohl
# der Tracktitel z. B. „Deutsch - Undefined - ASS“ enthält.
if kind == "subtitle" and lang["name"] == "Unbekannt":
if kind in {"audio", "subtitle"} and lang["name"] == "Unbekannt":
lang = language_from_text(tags.get("title")) or lang
if kind == "video" and not video: video = stream.get("codec_name")
elif kind == "audio":
audios.append(lang); details.append({"language": lang, "codec": stream.get("codec_name"), "channels": stream.get("channels"), "layout": stream.get("channel_layout"), "title": tags.get("title")})
source = "title" if norm_lang(tags.get("language"))["code"] == "und" and lang["code"] != "und" else "tag"
audios.append(lang); details.append({"language": lang, "language_source": source, "codec": stream.get("codec_name"), "channels": stream.get("channels"), "layout": stream.get("channel_layout"), "title": tags.get("title")})
elif kind == "subtitle": subs.append(lang)
subtitles = unique_langs(subs + external_subtitle_languages(path))
return {"scanner_version": SCANNER_VERSION, "audio_languages": unique_langs(audios), "subtitle_languages": subtitles, "external_subtitle_files": [str(file) for file in external_subtitle_files(path)], "audio_details": details, "video_codec": video or "—", "size": int((raw.get("format") or {}).get("size") or 0)}
def cached(path):
p = Path(path)
if not p.exists(): return {"error": f"Datei nicht gefunden: {p}", "pending": True}
mtime = p.stat().st_mtime; con = db(); row = con.execute("SELECT mtime,data FROM media_cache WHERE path=?", (str(p),)).fetchone(); con.close()
if not p.exists(): return {"error": f"Datei nicht gefunden: {p}", "pending": False}
mtime = p.stat().st_mtime; con = db(); row = con.execute("SELECT mtime,data,scanned_at FROM media_cache WHERE path=?", (str(p),)).fetchone(); con.close()
if row and float(row["mtime"]) == float(mtime):
data = json.loads(row["data"])
if data.get("error"):
scanned_at = datetime.fromisoformat(row["scanned_at"])
if scanned_at.tzinfo is None:
scanned_at = scanned_at.replace(tzinfo=timezone.utc)
if (datetime.now(timezone.utc) - scanned_at).total_seconds() >= SCAN_ERROR_RETRY_SECONDS:
return None
current_subtitles = [str(file) for file in external_subtitle_files(path)]
if data.get("scanner_version") == SCANNER_VERSION and "external_subtitle_files" in data and data["external_subtitle_files"] == current_subtitles:
return data
@@ -916,8 +945,8 @@ def scan_one(path):
try:
data = run_ffprobe(str(p)); log.info("Gescannt: %s", p)
except Exception as exc:
# Cache failed probes as completed attempts. A changed mtime will
# invalidate this entry automatically and trigger a retry later.
# Avoid immediate retry loops, but retry transient failures on a
# later sync even when the file itself has not changed.
data = {
"scanner_version": SCANNER_VERSION,
"audio_languages": [], "subtitle_languages": [],
@@ -926,7 +955,7 @@ def scan_one(path):
}
log.error("Scanfehler für %s: %s", p, error_summary(exc))
mtime = p.stat().st_mtime
con = db(); con.execute("INSERT INTO media_cache(path,mtime,data,scanned_at) VALUES(?,?,?,?) ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime,data=excluded.data,scanned_at=excluded.scanned_at", (str(p), mtime, json.dumps(data), datetime.utcnow().isoformat())); con.commit(); con.close()
con = db(); con.execute("INSERT INTO media_cache(path,mtime,data,scanned_at) VALUES(?,?,?,?) ON CONFLICT(path) DO UPDATE SET mtime=excluded.mtime,data=excluded.data,scanned_at=excluded.scanned_at", (str(p), mtime, json.dumps(data), datetime.now(timezone.utc).isoformat())); con.commit(); con.close()
return data
def format_size(size):
@@ -940,7 +969,8 @@ def language_state(audios):
names = {x["name"] for x in audios}
if {"Deutsch", "Englisch"} <= names: return {"class":"ok", "label":"DE + EN"}
if "Deutsch" in names: return {"class":"warn", "label":"Deutsch"}
if "Englisch" in names: return {"class":"bad", "label":"English only"}
if "Unbekannt" in names: return {"class":"neutral", "label":"Sprache prüfen"}
if names == {"Englisch"}: return {"class":"bad", "label":"English only"}
return {"class":"neutral", "label":"Andere"}
def has_german_track(row):
@@ -951,11 +981,35 @@ def is_german_review_candidate(row):
"""Only review media that exists or is currently being downloaded."""
return not row.get("missing") or bool(row.get("downloading"))
def needs_german_review(row):
# Failed and unfinished scans cannot establish that German is absent.
return is_german_review_candidate(row) and not row.get("pending") and not row.get("error") and not has_german_track(row)
def file_metadata_languages(media_file):
"""Read file-level Arr metadata, never the movie's original language."""
media_file = media_file or {}
values = (media_file.get("mediaInfo") or {}).get("audioLanguages") or []
if isinstance(values, str):
values = re.split(r"[,/;|]+", values)
languages = []
for value in values:
language = norm_lang(value)
if language["code"] in LANG_NAMES and language["code"] != "und":
languages.append(language)
if not languages:
for value in media_file.get("languages") or []:
language = norm_lang(value.get("name") if isinstance(value, dict) else value)
if language["code"] in LANG_NAMES and language["code"] != "und":
languages.append(language)
return [{**language, "source": "arr"} for language in unique_langs(languages)]
def media_row(name, year, media_file, remote, local, extra=None):
path = media_file.get("path") if media_file else None
if not path and media_file and media_file.get("relativePath"): path = str(Path(media_file.get("basePath") or "") / media_file["relativePath"])
local_path = map_path(path, remote, local); info = cached(local_path) if local_path else {"error":"Kein Dateipfad erhalten"}
info = info or {"pending": True}; audios = info.get("audio_languages", [])
info = dict(info or {"pending": True}); audios = info.get("audio_languages", [])
if media_file and (not audios or all(language["code"] == "und" for language in audios)):
audios = file_metadata_languages(media_file) or audios
if local_path:
info["subtitle_languages"] = unique_langs(info.get("subtitle_languages", []) + external_subtitle_languages(local_path))
downloading = bool((extra or {}).get("downloading"))
@@ -1025,6 +1079,7 @@ def start_scan(source, force=False, rows_snapshot=None, series_snapshot=None, pe
if SONARR_URL and SONARR_API_KEY:
for group in series_snapshot if series_snapshot is not None else sonarr_groups():
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and is_german_review_candidate(r) and not r.get("downloading") and not has_german_track(r) and (not pending_only or r.get("pending")))
paths = list(dict.fromkeys(paths))
with job_lock: jobs[source]["total"] = len(paths)
futures = [scan_executor.submit(scan_one, path) for path in paths]
for future in as_completed(futures):
@@ -1066,15 +1121,8 @@ def refresh_library_cache():
start_scan("radarr", force=True, rows_snapshot=rows)
if series and any(row.get("pending") for group in series for row in group["episodes"]):
start_scan("sonarr", force=True, series_snapshot=series)
needs_missing_scan = any(
row.get("local_path") != "—" and is_german_review_candidate(row) and row.get("pending") and not has_german_track(row)
for row in rows
) or any(
episode.get("local_path") != "—" and is_german_review_candidate(episode) and episode.get("pending") and not has_german_track(episode)
for group in series for episode in group["episodes"]
)
if needs_missing_scan:
start_scan("missing", force=True, rows_snapshot=rows, series_snapshot=series, pending_only=True)
# Radarr/Sonarr already scan every pending file. Starting a missing-German
# job here too would probe the same files twice in parallel.
log.info("Bibliotheks-Sync abgeschlossen: %s Filme, %s Serien", len(rows), len(series))
def background_library_sync():
@@ -1098,12 +1146,14 @@ def index():
rows = list(library_cache["rows"])
series = list(library_cache["series"])
error = library_cache["error"]
library_updated_at = library_cache["updated_at"]
library_ready = library_cache["ready"]
# Unscanned files have no language information yet and must not be shown
# as missing German while the background scanner is still processing them.
missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not row.get("pending") and not has_german_track(row)]
missing_german_movies = [row for row in rows if needs_german_review(row)]
missing_german_series = []
for group in series:
episodes = [episode for episode in group["episodes"] if is_german_review_candidate(episode) and not episode.get("pending") and not has_german_track(episode)]
episodes = [episode for episode in group["episodes"] if needs_german_review(episode)]
if episodes:
missing_german_series.append({**group, "episodes": episodes})
series_categories = [
@@ -1120,10 +1170,22 @@ def index():
"episode_files": sum(1 for group in series for row in group["episodes"] if not row.get("missing")),
"episode_missing": sum(1 for group in series for row in group["episodes"] if row.get("missing") and not row.get("downloading")),
}
existing = [row for row in rows + [episode for group in series for episode in group["episodes"]] if not row.get("missing")]
language_review = [row for row in existing if row.get("error") or any(language["code"] == "und" for language in row.get("audio_languages", []))]
dashboard_stats.update({
"german_tracks": sum(1 for row in existing if has_german_track(row)),
"unknown_audio": sum(1 for row in existing if any(language["code"] == "und" for language in row.get("audio_languages", []))),
"pending": sum(1 for row in existing if row.get("pending")),
"scan_errors": sum(1 for row in existing if row.get("error")),
"existing_total": len(existing),
})
return render_template("index.html", rows=rows, series=series, series_categories=series_categories,
missing_german_movies=missing_german_movies,
missing_german_series=missing_german_series, error=error,
dashboard_stats=dashboard_stats,
dashboard_stats=dashboard_stats,
language_names=sorted(set(LANG_NAMES.values())),
language_review=language_review[:5], library_updated_at=library_updated_at,
library_ready=library_ready, sync_interval_seconds=SYNC_INTERVAL_SECONDS,
sonarr_enabled=bool(SONARR_URL and SONARR_API_KEY), sab_enabled=bool(SAB_URL and SAB_API_KEY),
radarr_media_path=RADARR_MEDIA_PATH, local_media_path=LOCAL_MEDIA_PATH,
local_series_path=LOCAL_SERIES_PATH)