feat: add background library sync thread with configurable interval to eliminate browser dependency for automatic scanning

Add SYNC_INTERVAL_SECONDS environment variable with default 1800 (30 minutes). Add library_cache global dict with rows/series/updated_at/error/ready fields and library_cache_lock for thread-safe access. Extract refresh_library_cache() function to update Radarr/Sonarr data with error handling that preserves last good snapshot during outages. Add background_library_sync() daemon
This commit is contained in:
2026-08-25 20:11:38 +02:00
parent 7313eb96c3
commit 952797005f
4 changed files with 62 additions and 14 deletions
+3 -1
View File
@@ -83,15 +83,17 @@ Die Container müssen im gemeinsamen Docker-Netzwerk `media-stack_default` errei
## Scans und Performance ## Scans und Performance
Scans laufen nach dem Laden der Bibliotheksdaten im Hintergrund. Über den Button „Neu scannen“ kann zwischen Radarr, Sonarr, allen Quellen und ausschließlich den aktuellen Einträgen der Liste „Ohne Deutsch“ gewählt werden. Die automatische Scan-Häufigkeit wird in den Einstellungen festgelegt. Der Bibliotheks-Sync läuft als serverseitiger Hintergrund-Thread und benötigt keinen geöffneten Browser. Beim Start und anschließend nach `SYNC_INTERVAL_SECONDS` werden Radarr, Sonarr und die Sprachprüfung aktualisiert. Über den Button „Neu scannen“ kann weiterhin manuell zwischen Radarr, Sonarr, allen Quellen und ausschließlich den aktuellen Einträgen der Liste „Ohne Deutsch“ gewählt werden.
```env ```env
SCAN_WORKERS=4 SCAN_WORKERS=4
REQUEST_TIMEOUT=20 REQUEST_TIMEOUT=20
DB_PATH=/data/cache.db DB_PATH=/data/cache.db
SYNC_INTERVAL_SECONDS=1800
``` ```
`SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last. `SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last.
`SYNC_INTERVAL_SECONDS` steuert den serverseitigen Sync-Takt; `1800` entspricht 30 Minuten, `18000` fünf Stunden und `0` deaktiviert die periodische Wiederholung (der Start-Sync bleibt aktiv).
## Datastore, RAID und SMART ## Datastore, RAID und SMART
+56 -12
View File
@@ -48,6 +48,7 @@ STORAGE_CACHE_SECONDS = int(os.environ.get("STORAGE_CACHE_SECONDS", "900"))
SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"} SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"}
LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log") LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log")
LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6"))) LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6")))
SYNC_INTERVAL_SECONDS = max(0, int(os.environ.get("SYNC_INTERVAL_SECONDS", "1800")))
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s") logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
log = logging.getLogger("media-max") log = logging.getLogger("media-max")
@@ -90,6 +91,8 @@ raid_log_state = None
graphics_log_state = None graphics_log_state = None
graphics_process_log_state = None graphics_process_log_state = None
hardware_log_once = set() hardware_log_once = set()
library_cache_lock = threading.Lock()
library_cache = {"rows": [], "series": [], "updated_at": 0.0, "error": None, "ready": False}
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"} LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":""} FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":""}
@@ -960,7 +963,7 @@ def sonarr_groups():
"category":category_key, "category_label":category_label}) "category":category_key, "category_label":category_label})
return sorted(groups, key=lambda x: x["title"].lower()) return sorted(groups, key=lambda x: x["title"].lower())
def start_scan(source, force=False): def start_scan(source, force=False, rows_snapshot=None, series_snapshot=None):
with job_lock: with job_lock:
if jobs[source]["state"] == "running" or (jobs[source]["state"] in ("done", "error") and not force): return if jobs[source]["state"] == "running" or (jobs[source]["state"] in ("done", "error") and not force): return
jobs[source] = {"state":"running", "total":0, "done":0, "error":None} jobs[source] = {"state":"running", "total":0, "done":0, "error":None}
@@ -968,16 +971,16 @@ def start_scan(source, force=False):
try: try:
paths = [] paths = []
if source == "radarr": if source == "radarr":
for row in radarr_rows(): for row in rows_snapshot if rows_snapshot is not None else radarr_rows():
if row["local_path"] != "" and row["pending"]: paths.append(row["local_path"]) if row["local_path"] != "" and row["pending"]: paths.append(row["local_path"])
elif source == "sonarr": elif source == "sonarr":
for group in sonarr_groups(): for group in series_snapshot if series_snapshot is not None else sonarr_groups():
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and r["pending"]) paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and r["pending"])
else: else:
for row in radarr_rows(): for row in rows_snapshot if rows_snapshot is not None else radarr_rows():
if row["local_path"] != "" and is_german_review_candidate(row) and not has_german_track(row): paths.append(row["local_path"]) if row["local_path"] != "" and is_german_review_candidate(row) and not has_german_track(row): paths.append(row["local_path"])
if SONARR_URL and SONARR_API_KEY: if SONARR_URL and SONARR_API_KEY:
for group in sonarr_groups(): for group in series_snapshot if series_snapshot is not None else sonarr_groups():
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and is_german_review_candidate(r) and not has_german_track(r)) paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and is_german_review_candidate(r) and not has_german_track(r))
with job_lock: jobs[source]["total"] = len(paths) with job_lock: jobs[source]["total"] = len(paths)
futures = [scan_executor.submit(scan_one, path) for path in paths] futures = [scan_executor.submit(scan_one, path) for path in paths]
@@ -989,18 +992,59 @@ def start_scan(source, force=False):
log.error("%s-Scan fehlgeschlagen: %s", source, error_summary(exc)); jobs[source]["error"] = str(exc); jobs[source]["state"] = "error" log.error("%s-Scan fehlgeschlagen: %s", source, error_summary(exc)); jobs[source]["error"] = str(exc); jobs[source]["state"] = "error"
executor.submit(work) executor.submit(work)
@app.route("/") def refresh_library_cache():
def index(): """Refresh Radarr/Sonarr data outside the request thread."""
error = None; rows = []; series = [] rows = []
series = []
errors = []
radarr_ok = False
sonarr_ok = not (SONARR_URL and SONARR_API_KEY)
try: try:
rows = radarr_rows() rows = radarr_rows()
if any(row["pending"] for row in rows): start_scan("radarr") radarr_ok = True
except Exception as exc: error = str(exc) except Exception as exc:
errors.append(f"Radarr nicht erreichbar: {error_summary(exc)}")
if SONARR_URL and SONARR_API_KEY: if SONARR_URL and SONARR_API_KEY:
try: try:
series = sonarr_groups() series = sonarr_groups()
if any(row["pending"] for group in series for row in group["episodes"]): start_scan("sonarr") sonarr_ok = True
except Exception as exc: log.error("Sonarr nicht erreichbar: %s", error_summary(exc)); error = f"{error + ' | ' if error else ''}Sonarr nicht erreichbar" except Exception as exc:
errors.append(f"Sonarr nicht erreichbar: {error_summary(exc)}")
with library_cache_lock:
# Keep the last good snapshot during a short service/DNS outage.
if not radarr_ok:
rows = library_cache["rows"]
if not sonarr_ok:
series = library_cache["series"]
library_cache.update({"rows": rows, "series": series, "updated_at": time.time(), "error": " | ".join(errors) or None, "ready": True})
if rows and any(row.get("pending") for row in rows):
start_scan("radarr", force=True, rows_snapshot=rows)
if series and any(row.get("pending") for group in series for row in group["episodes"]):
start_scan("sonarr", force=True, series_snapshot=series)
if rows or series:
start_scan("missing", force=True, rows_snapshot=rows, series_snapshot=series)
log.info("Bibliotheks-Sync abgeschlossen: %s Filme, %s Serien", len(rows), len(series))
def background_library_sync():
"""Keep library data and media metadata current without a browser tab."""
log.info("Hintergrund-Sync gestartet (Intervall: %ss)", SYNC_INTERVAL_SECONDS or "deaktiviert")
while True:
try:
refresh_library_cache()
except Exception as exc:
log.error("Hintergrund-Sync fehlgeschlagen: %s", error_summary(exc))
if not SYNC_INTERVAL_SECONDS:
return
time.sleep(SYNC_INTERVAL_SECONDS)
threading.Thread(target=background_library_sync, name="library-sync", daemon=True).start()
@app.route("/")
def index():
with library_cache_lock:
rows = list(library_cache["rows"])
series = list(library_cache["series"])
error = library_cache["error"]
missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not has_german_track(row)] missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not has_german_track(row)]
missing_german_series = [] missing_german_series = []
for group in series: for group in series:
+2
View File
@@ -20,6 +20,8 @@ services:
environment: environment:
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all} NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility} NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
# Serverseitiger Bibliotheks-Sync; 1800 = alle 30 Minuten, 0 = nur beim Start.
SYNC_INTERVAL_SECONDS: ${SYNC_INTERVAL_SECONDS:-1800}
volumes: volumes:
# Host-Pfad deiner Film-Library -> interner Dashboard-Pfad # Host-Pfad deiner Film-Library -> interner Dashboard-Pfad
- ${FILM_MEDIA_VOLUME}:/media/filme:ro - ${FILM_MEDIA_VOLUME}:/media/filme:ro
+1 -1
View File
@@ -101,7 +101,7 @@ document.querySelector('#rescan').onclick=async()=>{const b=document.querySelect
</script> </script>
<script> <script>
const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)} const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)}
let autoScanTimer=null;function configureAutoScan(){if(autoScanTimer)clearInterval(autoScanTimer);const interval=Number(localStorage.getItem('auto-scan-interval')||0);if(!interval)return;autoScanTimer=setInterval(()=>{['radarr','sonarr'].forEach(source=>{if(document.querySelector(`#${source}`))fetch(`/api/rescan?source=${source}`,{method:'POST'})})},interval)}function saveAutoScan(){localStorage.setItem('auto-scan-interval',scanSelect.value);configureAutoScan()}scanSelect.value=localStorage.getItem('auto-scan-interval')||'0';configureAutoScan();document.querySelector('#settingsSave').addEventListener('click',saveAutoScan); function configureAutoScan(){}function saveAutoScan(){localStorage.setItem('auto-scan-interval',scanSelect.value)}scanSelect.value=localStorage.getItem('auto-scan-interval')||'0';document.querySelector('#settingsSave').addEventListener('click',saveAutoScan);
const seasonZeroLabel=document.createElement('label');seasonZeroLabel.className='setting-toggle';seasonZeroLabel.htmlFor='showSeasonZero';seasonZeroLabel.innerHTML='<input type="checkbox" id="showSeasonZero"><span>Staffel 0 anzeigen</span>';const seasonZeroHelp=document.createElement('span');seasonZeroHelp.className='setting-help';seasonZeroHelp.textContent='Gilt für Sonarr und die Liste Ohne Deutsch.';if(settingsActions){settingsActions.before(seasonZeroLabel,seasonZeroHelp)} const seasonZeroLabel=document.createElement('label');seasonZeroLabel.className='setting-toggle';seasonZeroLabel.htmlFor='showSeasonZero';seasonZeroLabel.innerHTML='<input type="checkbox" id="showSeasonZero"><span>Staffel 0 anzeigen</span>';const seasonZeroHelp=document.createElement('span');seasonZeroHelp.className='setting-help';seasonZeroHelp.textContent='Gilt für Sonarr und die Liste Ohne Deutsch.';if(settingsActions){settingsActions.before(seasonZeroLabel,seasonZeroHelp)}
function applySeasonZeroPreference(){const show=localStorage.getItem('show-season-zero')==='true';const rows=[...document.querySelectorAll('#sonarr tbody tr:not(.season-divider),#missing .missing-series tbody tr:not(.season-divider)')];rows.forEach(row=>{const title=row.querySelector('.title')?.textContent.trim()||'';row.classList.toggle('season-zero-hidden',!show&&/^S00E/i.test(title))});document.querySelectorAll('#sonarr details.series,#missing details.missing-series').forEach(detail=>{const episodeRows=[...detail.querySelectorAll('tbody tr:not(.season-divider)')];detail.classList.toggle('season-zero-empty',episodeRows.length>0&&episodeRows.every(row=>row.classList.contains('season-zero-hidden')))});const missingCount=document.querySelector('#missingCount'),missingNavCount=document.querySelector('#missingNavCount');if(missingCount||missingNavCount){const movies=document.querySelectorAll('#missingMovies tbody tr:not(.hidden)').length,episodes=document.querySelectorAll('#missing .missing-series tbody tr:not(.season-divider):not(.season-zero-hidden)').length,total=movies+episodes;if(missingCount)missingCount.textContent=`${total} Einträge ohne Deutsch`;if(missingNavCount)missingNavCount.textContent=total}const hint=document.querySelector('#seasonZeroHint');if(hint)hint.textContent=show?'Vorhanden oder wird gerade geladen · Staffel 0 eingeblendet':'Vorhanden oder wird gerade geladen · Staffel 0 ausgeblendet'}const seasonZeroToggle=document.querySelector('#showSeasonZero');if(seasonZeroToggle){seasonZeroToggle.checked=localStorage.getItem('show-season-zero')==='true';seasonZeroToggle.addEventListener('change',()=>{localStorage.setItem('show-season-zero',seasonZeroToggle.checked?'true':'false');applySeasonZeroPreference()})}applySeasonZeroPreference(); function applySeasonZeroPreference(){const show=localStorage.getItem('show-season-zero')==='true';const rows=[...document.querySelectorAll('#sonarr tbody tr:not(.season-divider),#missing .missing-series tbody tr:not(.season-divider)')];rows.forEach(row=>{const title=row.querySelector('.title')?.textContent.trim()||'';row.classList.toggle('season-zero-hidden',!show&&/^S00E/i.test(title))});document.querySelectorAll('#sonarr details.series,#missing details.missing-series').forEach(detail=>{const episodeRows=[...detail.querySelectorAll('tbody tr:not(.season-divider)')];detail.classList.toggle('season-zero-empty',episodeRows.length>0&&episodeRows.every(row=>row.classList.contains('season-zero-hidden')))});const missingCount=document.querySelector('#missingCount'),missingNavCount=document.querySelector('#missingNavCount');if(missingCount||missingNavCount){const movies=document.querySelectorAll('#missingMovies tbody tr:not(.hidden)').length,episodes=document.querySelectorAll('#missing .missing-series tbody tr:not(.season-divider):not(.season-zero-hidden)').length,total=movies+episodes;if(missingCount)missingCount.textContent=`${total} Einträge ohne Deutsch`;if(missingNavCount)missingNavCount.textContent=total}const hint=document.querySelector('#seasonZeroHint');if(hint)hint.textContent=show?'Vorhanden oder wird gerade geladen · Staffel 0 eingeblendet':'Vorhanden oder wird gerade geladen · Staffel 0 ausgeblendet'}const seasonZeroToggle=document.querySelector('#showSeasonZero');if(seasonZeroToggle){seasonZeroToggle.checked=localStorage.getItem('show-season-zero')==='true';seasonZeroToggle.addEventListener('change',()=>{localStorage.setItem('show-season-zero',seasonZeroToggle.checked?'true':'false');applySeasonZeroPreference()})}applySeasonZeroPreference();
function syncDashboardMissingCounter(){const visibleMissing=document.querySelectorAll('#missingMovies tbody tr:not(.hidden),#missing .missing-series tbody tr:not(.season-zero-hidden):not(.season-divider)').length;const dashboardMissing=document.querySelector('#dashboardMissing');if(dashboardMissing)dashboardMissing.textContent=visibleMissing}syncDashboardMissingCounter();document.querySelector('#showSeasonZero')?.addEventListener('change',()=>setTimeout(syncDashboardMissingCounter,0)); function syncDashboardMissingCounter(){const visibleMissing=document.querySelectorAll('#missingMovies tbody tr:not(.hidden),#missing .missing-series tbody tr:not(.season-zero-hidden):not(.season-divider)').length;const dashboardMissing=document.querySelector('#dashboardMissing');if(dashboardMissing)dashboardMissing.textContent=visibleMissing}syncDashboardMissingCounter();document.querySelector('#showSeasonZero')?.addEventListener('change',()=>setTimeout(syncDashboardMissingCounter,0));