feat: add background library sync thread with configurable interval to eliminate browser dependency for automatic scanning
Add SYNC_INTERVAL_SECONDS environment variable with default 1800 (30 minutes). Add library_cache global dict with rows/series/updated_at/error/ready fields and library_cache_lock for thread-safe access. Extract refresh_library_cache() function to update Radarr/Sonarr data with error handling that preserves last good snapshot during outages. Add background_library_sync() daemon
This commit is contained in:
@@ -83,15 +83,17 @@ Die Container müssen im gemeinsamen Docker-Netzwerk `media-stack_default` errei
|
|||||||
|
|
||||||
## Scans und Performance
|
## Scans und Performance
|
||||||
|
|
||||||
Scans laufen nach dem Laden der Bibliotheksdaten im Hintergrund. Über den Button „Neu scannen“ kann zwischen Radarr, Sonarr, allen Quellen und ausschließlich den aktuellen Einträgen der Liste „Ohne Deutsch“ gewählt werden. Die automatische Scan-Häufigkeit wird in den Einstellungen festgelegt.
|
Der Bibliotheks-Sync läuft als serverseitiger Hintergrund-Thread und benötigt keinen geöffneten Browser. Beim Start und anschließend nach `SYNC_INTERVAL_SECONDS` werden Radarr, Sonarr und die Sprachprüfung aktualisiert. Über den Button „Neu scannen“ kann weiterhin manuell zwischen Radarr, Sonarr, allen Quellen und ausschließlich den aktuellen Einträgen der Liste „Ohne Deutsch“ gewählt werden.
|
||||||
|
|
||||||
```env
|
```env
|
||||||
SCAN_WORKERS=4
|
SCAN_WORKERS=4
|
||||||
REQUEST_TIMEOUT=20
|
REQUEST_TIMEOUT=20
|
||||||
DB_PATH=/data/cache.db
|
DB_PATH=/data/cache.db
|
||||||
|
SYNC_INTERVAL_SECONDS=1800
|
||||||
```
|
```
|
||||||
|
|
||||||
`SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last.
|
`SCAN_WORKERS` steuert die Anzahl paralleler `ffprobe`-Jobs. Höhere Werte beschleunigen große Bibliotheken, erhöhen aber die I/O-Last.
|
||||||
|
`SYNC_INTERVAL_SECONDS` steuert den serverseitigen Sync-Takt; `1800` entspricht 30 Minuten, `18000` fünf Stunden und `0` deaktiviert die periodische Wiederholung (der Start-Sync bleibt aktiv).
|
||||||
|
|
||||||
## Datastore, RAID und SMART
|
## Datastore, RAID und SMART
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ STORAGE_CACHE_SECONDS = int(os.environ.get("STORAGE_CACHE_SECONDS", "900"))
|
|||||||
SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"}
|
SMART_ENABLED = os.environ.get("SMART_ENABLED", "false").lower() in {"1", "true", "yes", "on"}
|
||||||
LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log")
|
LOG_FILE = os.environ.get("LOG_FILE", "/data/media-max.log")
|
||||||
LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6")))
|
LOG_RETENTION_HOURS = max(1, int(os.environ.get("LOG_RETENTION_HOURS", "6")))
|
||||||
|
SYNC_INTERVAL_SECONDS = max(0, int(os.environ.get("SYNC_INTERVAL_SECONDS", "1800")))
|
||||||
|
|
||||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
|
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s")
|
||||||
log = logging.getLogger("media-max")
|
log = logging.getLogger("media-max")
|
||||||
@@ -90,6 +91,8 @@ raid_log_state = None
|
|||||||
graphics_log_state = None
|
graphics_log_state = None
|
||||||
graphics_process_log_state = None
|
graphics_process_log_state = None
|
||||||
hardware_log_once = set()
|
hardware_log_once = set()
|
||||||
|
library_cache_lock = threading.Lock()
|
||||||
|
library_cache = {"rows": [], "series": [], "updated_at": 0.0, "error": None, "ready": False}
|
||||||
|
|
||||||
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
|
LANG_NAMES = {"de":"Deutsch", "deu":"Deutsch", "ger":"Deutsch", "en":"Englisch", "eng":"Englisch", "ja":"Japanisch", "jpn":"Japanisch", "ko":"Koreanisch", "kor":"Koreanisch", "fr":"Französisch", "fra":"Französisch", "fre":"Französisch", "es":"Spanisch", "spa":"Spanisch", "it":"Italienisch", "ita":"Italienisch", "ru":"Russisch", "rus":"Russisch", "zh":"Chinesisch", "zho":"Chinesisch", "chi":"Chinesisch", "und":"Unbekannt", "":"Unbekannt"}
|
||||||
FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"}
|
FLAGS = {"Deutsch":"🇩🇪", "Englisch":"🇬🇧", "Japanisch":"🇯🇵", "Koreanisch":"🇰🇷", "Französisch":"🇫🇷", "Spanisch":"🇪🇸", "Italienisch":"🇮🇹", "Russisch":"🇷🇺", "Chinesisch":"🇨🇳", "Unbekannt":"❓"}
|
||||||
@@ -960,7 +963,7 @@ def sonarr_groups():
|
|||||||
"category":category_key, "category_label":category_label})
|
"category":category_key, "category_label":category_label})
|
||||||
return sorted(groups, key=lambda x: x["title"].lower())
|
return sorted(groups, key=lambda x: x["title"].lower())
|
||||||
|
|
||||||
def start_scan(source, force=False):
|
def start_scan(source, force=False, rows_snapshot=None, series_snapshot=None):
|
||||||
with job_lock:
|
with job_lock:
|
||||||
if jobs[source]["state"] == "running" or (jobs[source]["state"] in ("done", "error") and not force): return
|
if jobs[source]["state"] == "running" or (jobs[source]["state"] in ("done", "error") and not force): return
|
||||||
jobs[source] = {"state":"running", "total":0, "done":0, "error":None}
|
jobs[source] = {"state":"running", "total":0, "done":0, "error":None}
|
||||||
@@ -968,16 +971,16 @@ def start_scan(source, force=False):
|
|||||||
try:
|
try:
|
||||||
paths = []
|
paths = []
|
||||||
if source == "radarr":
|
if source == "radarr":
|
||||||
for row in radarr_rows():
|
for row in rows_snapshot if rows_snapshot is not None else radarr_rows():
|
||||||
if row["local_path"] != "—" and row["pending"]: paths.append(row["local_path"])
|
if row["local_path"] != "—" and row["pending"]: paths.append(row["local_path"])
|
||||||
elif source == "sonarr":
|
elif source == "sonarr":
|
||||||
for group in sonarr_groups():
|
for group in series_snapshot if series_snapshot is not None else sonarr_groups():
|
||||||
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and r["pending"])
|
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and r["pending"])
|
||||||
else:
|
else:
|
||||||
for row in radarr_rows():
|
for row in rows_snapshot if rows_snapshot is not None else radarr_rows():
|
||||||
if row["local_path"] != "—" and is_german_review_candidate(row) and not has_german_track(row): paths.append(row["local_path"])
|
if row["local_path"] != "—" and is_german_review_candidate(row) and not has_german_track(row): paths.append(row["local_path"])
|
||||||
if SONARR_URL and SONARR_API_KEY:
|
if SONARR_URL and SONARR_API_KEY:
|
||||||
for group in sonarr_groups():
|
for group in series_snapshot if series_snapshot is not None else sonarr_groups():
|
||||||
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and is_german_review_candidate(r) and not has_german_track(r))
|
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "—" and is_german_review_candidate(r) and not has_german_track(r))
|
||||||
with job_lock: jobs[source]["total"] = len(paths)
|
with job_lock: jobs[source]["total"] = len(paths)
|
||||||
futures = [scan_executor.submit(scan_one, path) for path in paths]
|
futures = [scan_executor.submit(scan_one, path) for path in paths]
|
||||||
@@ -989,18 +992,59 @@ def start_scan(source, force=False):
|
|||||||
log.error("%s-Scan fehlgeschlagen: %s", source, error_summary(exc)); jobs[source]["error"] = str(exc); jobs[source]["state"] = "error"
|
log.error("%s-Scan fehlgeschlagen: %s", source, error_summary(exc)); jobs[source]["error"] = str(exc); jobs[source]["state"] = "error"
|
||||||
executor.submit(work)
|
executor.submit(work)
|
||||||
|
|
||||||
@app.route("/")
|
def refresh_library_cache():
|
||||||
def index():
|
"""Refresh Radarr/Sonarr data outside the request thread."""
|
||||||
error = None; rows = []; series = []
|
rows = []
|
||||||
|
series = []
|
||||||
|
errors = []
|
||||||
|
radarr_ok = False
|
||||||
|
sonarr_ok = not (SONARR_URL and SONARR_API_KEY)
|
||||||
try:
|
try:
|
||||||
rows = radarr_rows()
|
rows = radarr_rows()
|
||||||
if any(row["pending"] for row in rows): start_scan("radarr")
|
radarr_ok = True
|
||||||
except Exception as exc: error = str(exc)
|
except Exception as exc:
|
||||||
|
errors.append(f"Radarr nicht erreichbar: {error_summary(exc)}")
|
||||||
if SONARR_URL and SONARR_API_KEY:
|
if SONARR_URL and SONARR_API_KEY:
|
||||||
try:
|
try:
|
||||||
series = sonarr_groups()
|
series = sonarr_groups()
|
||||||
if any(row["pending"] for group in series for row in group["episodes"]): start_scan("sonarr")
|
sonarr_ok = True
|
||||||
except Exception as exc: log.error("Sonarr nicht erreichbar: %s", error_summary(exc)); error = f"{error + ' | ' if error else ''}Sonarr nicht erreichbar"
|
except Exception as exc:
|
||||||
|
errors.append(f"Sonarr nicht erreichbar: {error_summary(exc)}")
|
||||||
|
with library_cache_lock:
|
||||||
|
# Keep the last good snapshot during a short service/DNS outage.
|
||||||
|
if not radarr_ok:
|
||||||
|
rows = library_cache["rows"]
|
||||||
|
if not sonarr_ok:
|
||||||
|
series = library_cache["series"]
|
||||||
|
library_cache.update({"rows": rows, "series": series, "updated_at": time.time(), "error": " | ".join(errors) or None, "ready": True})
|
||||||
|
if rows and any(row.get("pending") for row in rows):
|
||||||
|
start_scan("radarr", force=True, rows_snapshot=rows)
|
||||||
|
if series and any(row.get("pending") for group in series for row in group["episodes"]):
|
||||||
|
start_scan("sonarr", force=True, series_snapshot=series)
|
||||||
|
if rows or series:
|
||||||
|
start_scan("missing", force=True, rows_snapshot=rows, series_snapshot=series)
|
||||||
|
log.info("Bibliotheks-Sync abgeschlossen: %s Filme, %s Serien", len(rows), len(series))
|
||||||
|
|
||||||
|
def background_library_sync():
|
||||||
|
"""Keep library data and media metadata current without a browser tab."""
|
||||||
|
log.info("Hintergrund-Sync gestartet (Intervall: %ss)", SYNC_INTERVAL_SECONDS or "deaktiviert")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
refresh_library_cache()
|
||||||
|
except Exception as exc:
|
||||||
|
log.error("Hintergrund-Sync fehlgeschlagen: %s", error_summary(exc))
|
||||||
|
if not SYNC_INTERVAL_SECONDS:
|
||||||
|
return
|
||||||
|
time.sleep(SYNC_INTERVAL_SECONDS)
|
||||||
|
|
||||||
|
threading.Thread(target=background_library_sync, name="library-sync", daemon=True).start()
|
||||||
|
|
||||||
|
@app.route("/")
|
||||||
|
def index():
|
||||||
|
with library_cache_lock:
|
||||||
|
rows = list(library_cache["rows"])
|
||||||
|
series = list(library_cache["series"])
|
||||||
|
error = library_cache["error"]
|
||||||
missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not has_german_track(row)]
|
missing_german_movies = [row for row in rows if is_german_review_candidate(row) and not has_german_track(row)]
|
||||||
missing_german_series = []
|
missing_german_series = []
|
||||||
for group in series:
|
for group in series:
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
NVIDIA_VISIBLE_DEVICES: ${NVIDIA_VISIBLE_DEVICES:-all}
|
||||||
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
NVIDIA_DRIVER_CAPABILITIES: ${NVIDIA_DRIVER_CAPABILITIES:-compute,utility}
|
||||||
|
# Serverseitiger Bibliotheks-Sync; 1800 = alle 30 Minuten, 0 = nur beim Start.
|
||||||
|
SYNC_INTERVAL_SECONDS: ${SYNC_INTERVAL_SECONDS:-1800}
|
||||||
volumes:
|
volumes:
|
||||||
# Host-Pfad deiner Film-Library -> interner Dashboard-Pfad
|
# Host-Pfad deiner Film-Library -> interner Dashboard-Pfad
|
||||||
- ${FILM_MEDIA_VOLUME}:/media/filme:ro
|
- ${FILM_MEDIA_VOLUME}:/media/filme:ro
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ document.querySelector('#rescan').onclick=async()=>{const b=document.querySelect
|
|||||||
</script>
|
</script>
|
||||||
<script>
|
<script>
|
||||||
const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)}
|
const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)}
|
||||||
let autoScanTimer=null;function configureAutoScan(){if(autoScanTimer)clearInterval(autoScanTimer);const interval=Number(localStorage.getItem('auto-scan-interval')||0);if(!interval)return;autoScanTimer=setInterval(()=>{['radarr','sonarr'].forEach(source=>{if(document.querySelector(`#${source}`))fetch(`/api/rescan?source=${source}`,{method:'POST'})})},interval)}function saveAutoScan(){localStorage.setItem('auto-scan-interval',scanSelect.value);configureAutoScan()}scanSelect.value=localStorage.getItem('auto-scan-interval')||'0';configureAutoScan();document.querySelector('#settingsSave').addEventListener('click',saveAutoScan);
|
function configureAutoScan(){}function saveAutoScan(){localStorage.setItem('auto-scan-interval',scanSelect.value)}scanSelect.value=localStorage.getItem('auto-scan-interval')||'0';document.querySelector('#settingsSave').addEventListener('click',saveAutoScan);
|
||||||
const seasonZeroLabel=document.createElement('label');seasonZeroLabel.className='setting-toggle';seasonZeroLabel.htmlFor='showSeasonZero';seasonZeroLabel.innerHTML='<input type="checkbox" id="showSeasonZero"><span>Staffel 0 anzeigen</span>';const seasonZeroHelp=document.createElement('span');seasonZeroHelp.className='setting-help';seasonZeroHelp.textContent='Gilt für Sonarr und die Liste Ohne Deutsch.';if(settingsActions){settingsActions.before(seasonZeroLabel,seasonZeroHelp)}
|
const seasonZeroLabel=document.createElement('label');seasonZeroLabel.className='setting-toggle';seasonZeroLabel.htmlFor='showSeasonZero';seasonZeroLabel.innerHTML='<input type="checkbox" id="showSeasonZero"><span>Staffel 0 anzeigen</span>';const seasonZeroHelp=document.createElement('span');seasonZeroHelp.className='setting-help';seasonZeroHelp.textContent='Gilt für Sonarr und die Liste Ohne Deutsch.';if(settingsActions){settingsActions.before(seasonZeroLabel,seasonZeroHelp)}
|
||||||
function applySeasonZeroPreference(){const show=localStorage.getItem('show-season-zero')==='true';const rows=[...document.querySelectorAll('#sonarr tbody tr:not(.season-divider),#missing .missing-series tbody tr:not(.season-divider)')];rows.forEach(row=>{const title=row.querySelector('.title')?.textContent.trim()||'';row.classList.toggle('season-zero-hidden',!show&&/^S00E/i.test(title))});document.querySelectorAll('#sonarr details.series,#missing details.missing-series').forEach(detail=>{const episodeRows=[...detail.querySelectorAll('tbody tr:not(.season-divider)')];detail.classList.toggle('season-zero-empty',episodeRows.length>0&&episodeRows.every(row=>row.classList.contains('season-zero-hidden')))});const missingCount=document.querySelector('#missingCount'),missingNavCount=document.querySelector('#missingNavCount');if(missingCount||missingNavCount){const movies=document.querySelectorAll('#missingMovies tbody tr:not(.hidden)').length,episodes=document.querySelectorAll('#missing .missing-series tbody tr:not(.season-divider):not(.season-zero-hidden)').length,total=movies+episodes;if(missingCount)missingCount.textContent=`${total} Einträge ohne Deutsch`;if(missingNavCount)missingNavCount.textContent=total}const hint=document.querySelector('#seasonZeroHint');if(hint)hint.textContent=show?'Vorhanden oder wird gerade geladen · Staffel 0 eingeblendet':'Vorhanden oder wird gerade geladen · Staffel 0 ausgeblendet'}const seasonZeroToggle=document.querySelector('#showSeasonZero');if(seasonZeroToggle){seasonZeroToggle.checked=localStorage.getItem('show-season-zero')==='true';seasonZeroToggle.addEventListener('change',()=>{localStorage.setItem('show-season-zero',seasonZeroToggle.checked?'true':'false');applySeasonZeroPreference()})}applySeasonZeroPreference();
|
function applySeasonZeroPreference(){const show=localStorage.getItem('show-season-zero')==='true';const rows=[...document.querySelectorAll('#sonarr tbody tr:not(.season-divider),#missing .missing-series tbody tr:not(.season-divider)')];rows.forEach(row=>{const title=row.querySelector('.title')?.textContent.trim()||'';row.classList.toggle('season-zero-hidden',!show&&/^S00E/i.test(title))});document.querySelectorAll('#sonarr details.series,#missing details.missing-series').forEach(detail=>{const episodeRows=[...detail.querySelectorAll('tbody tr:not(.season-divider)')];detail.classList.toggle('season-zero-empty',episodeRows.length>0&&episodeRows.every(row=>row.classList.contains('season-zero-hidden')))});const missingCount=document.querySelector('#missingCount'),missingNavCount=document.querySelector('#missingNavCount');if(missingCount||missingNavCount){const movies=document.querySelectorAll('#missingMovies tbody tr:not(.hidden)').length,episodes=document.querySelectorAll('#missing .missing-series tbody tr:not(.season-divider):not(.season-zero-hidden)').length,total=movies+episodes;if(missingCount)missingCount.textContent=`${total} Einträge ohne Deutsch`;if(missingNavCount)missingNavCount.textContent=total}const hint=document.querySelector('#seasonZeroHint');if(hint)hint.textContent=show?'Vorhanden oder wird gerade geladen · Staffel 0 eingeblendet':'Vorhanden oder wird gerade geladen · Staffel 0 ausgeblendet'}const seasonZeroToggle=document.querySelector('#showSeasonZero');if(seasonZeroToggle){seasonZeroToggle.checked=localStorage.getItem('show-season-zero')==='true';seasonZeroToggle.addEventListener('change',()=>{localStorage.setItem('show-season-zero',seasonZeroToggle.checked?'true':'false');applySeasonZeroPreference()})}applySeasonZeroPreference();
|
||||||
function syncDashboardMissingCounter(){const visibleMissing=document.querySelectorAll('#missingMovies tbody tr:not(.hidden),#missing .missing-series tbody tr:not(.season-zero-hidden):not(.season-divider)').length;const dashboardMissing=document.querySelector('#dashboardMissing');if(dashboardMissing)dashboardMissing.textContent=visibleMissing}syncDashboardMissingCounter();document.querySelector('#showSeasonZero')?.addEventListener('change',()=>setTimeout(syncDashboardMissingCounter,0));
|
function syncDashboardMissingCounter(){const visibleMissing=document.querySelectorAll('#missingMovies tbody tr:not(.hidden),#missing .missing-series tbody tr:not(.season-zero-hidden):not(.season-divider)').length;const dashboardMissing=document.querySelector('#dashboardMissing');if(dashboardMissing)dashboardMissing.textContent=visibleMissing}syncDashboardMissingCounter();document.querySelector('#showSeasonZero')?.addEventListener('change',()=>setTimeout(syncDashboardMissingCounter,0));
|
||||||
|
|||||||
Reference in New Issue
Block a user