feat: add parallel file scanning with dedicated executor and support for scanning all sources simultaneously

Add scan_executor ThreadPoolExecutor with 4 workers for parallel file scanning. Import as_completed from concurrent.futures to process scan results. Replace sequential scan_one() loop with parallel futures submission and result collection using as_completed().

Update /api/rescan endpoint to accept "all" source parameter for scanning both Radarr and Sonarr simultaneously. Fix cache deletion
This commit is contained in:
2026-08-14 13:22:15 +02:00
parent 644c54a6f6
commit b5365170fb
2 changed files with 19 additions and 11 deletions
+15 -7
View File
@@ -7,7 +7,7 @@ import threading
import time import time
import shutil import shutil
import re import re
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
@@ -31,6 +31,7 @@ logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), format="%
log = logging.getLogger("radarr-language-dashboard") log = logging.getLogger("radarr-language-dashboard")
app = Flask(__name__) app = Flask(__name__)
executor = ThreadPoolExecutor(max_workers=int(os.environ.get("SCAN_WORKERS", "2"))) executor = ThreadPoolExecutor(max_workers=int(os.environ.get("SCAN_WORKERS", "2")))
scan_executor = ThreadPoolExecutor(max_workers=int(os.environ.get("SCAN_WORKERS", "4")))
jobs = {"radarr": {"state": "idle", "total": 0, "done": 0, "error": None}, "sonarr": {"state": "idle", "total": 0, "done": 0, "error": None}} jobs = {"radarr": {"state": "idle", "total": 0, "done": 0, "error": None}, "sonarr": {"state": "idle", "total": 0, "done": 0, "error": None}}
job_lock = threading.Lock() job_lock = threading.Lock()
system_lock = threading.Lock() system_lock = threading.Lock()
@@ -283,8 +284,9 @@ def start_scan(source, force=False):
for group in sonarr_groups(): for group in sonarr_groups():
paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and r["pending"]) paths.extend(r["local_path"] for r in group["episodes"] if r["local_path"] != "" and r["pending"])
with job_lock: jobs[source]["total"] = len(paths) with job_lock: jobs[source]["total"] = len(paths)
for path in paths: futures = [scan_executor.submit(scan_one, path) for path in paths]
scan_one(path) for future in as_completed(futures):
future.result()
with job_lock: jobs[source]["done"] += 1 with job_lock: jobs[source]["done"] += 1
with job_lock: jobs[source]["state"] = "done" with job_lock: jobs[source]["state"] = "done"
except Exception as exc: except Exception as exc:
@@ -329,11 +331,17 @@ def index():
@app.post("/api/rescan") @app.post("/api/rescan")
def api_rescan(): def api_rescan():
source = request.args.get("source", "radarr") source = request.args.get("source", "radarr")
if source not in jobs: return jsonify({"ok":False, "error":"Unbekannte Quelle"}), 400 if source not in (*jobs.keys(), "all"): return jsonify({"ok":False, "error":"Unbekannte Quelle"}), 400
con = db() con = db()
if source == "radarr": con.execute("DELETE FROM media_cache") if source in ("radarr", "all"): con.execute("DELETE FROM media_cache WHERE path NOT LIKE ?", (LOCAL_SERIES_PATH.rstrip("/") + "/%",))
else: con.execute("DELETE FROM media_cache WHERE path LIKE ?", (LOCAL_SERIES_PATH.rstrip("/") + "/%",)) if source in ("sonarr", "all"): con.execute("DELETE FROM media_cache WHERE path LIKE ?", (LOCAL_SERIES_PATH.rstrip("/") + "/%",))
con.commit(); con.close(); start_scan(source, force=True) con.commit(); con.close()
if source == "all":
start_scan("radarr", force=True)
if SONARR_URL and SONARR_API_KEY:
start_scan("sonarr", force=True)
else:
start_scan(source, force=True)
return jsonify({"ok":True}) return jsonify({"ok":True})
@app.get("/api/scan-status") @app.get("/api/scan-status")
+4 -4
View File
@@ -9,8 +9,8 @@
.settings-modal .setting-help{display:block;margin:-3px 0 12px;color:var(--muted);font-size:11px}.settings-modal .setting-select{width:100%;height:38px}.pwa-install{display:none} .settings-modal .setting-help{display:block;margin:-3px 0 12px;color:var(--muted);font-size:11px}.settings-modal .setting-select{width:100%;height:38px}.pwa-install{display:none}
@media(max-width:700px){body{font-size:12px;padding-bottom:env(safe-area-inset-bottom)}header{height:auto;min-height:62px;padding:9px 12px;gap:9px;align-items:center}.brand-mark,.brand-mark svg{width:30px;height:30px}.brand-name{font-size:14px}.head-title{font-size:12px}.head-sub{font-size:10px}.actions{gap:4px}.actions button{height:34px;padding:0 8px}.settings-button{font-size:15px}.actions #rescan{font-size:0}.actions #rescan::after{content:'↻';font-size:16px}main{padding:10px 10px 24px}.nav{margin-bottom:10px;padding:4px;overflow-x:auto;scrollbar-width:none}.nav::-webkit-scrollbar{display:none}.nav button{flex:0 0 auto;white-space:nowrap;padding:8px 10px;font-size:11px}.toolbar{grid-template-columns:1fr;gap:7px;padding:8px}.toolbar input,.toolbar select{width:100%;height:38px}.stats{gap:6px}.stat{padding:6px 8px}.tablewrap{overflow-x:auto;border-radius:8px}table{min-width:860px}.dashboard-hero{margin-bottom:10px;padding:18px;border-radius:11px}.dashboard-hero h1{font-size:21px}.dashboard-hero p{font-size:11px;line-height:1.4}.overview-grid{grid-template-columns:1fr 1fr;gap:7px}.overview-card{min-height:84px;padding:12px 10px;gap:8px}.overview-icon{width:32px;height:32px;font-size:18px}.overview-card strong{font-size:18px}.overview-card span:not(.overview-icon){font-size:9px}.overview-card small{font-size:9px}.dashboard-columns{gap:8px}.system-panel,.service-panel{padding:14px;border-radius:10px}.system-meters{gap:12px}.panel-heading{margin-bottom:13px}.panel-heading h2{font-size:14px}.panel-heading p{font-size:10px}.disk-row{grid-template-columns:82px 1fr auto;gap:7px}.disk-values{font-size:10px}.download-hero{padding:18px;margin-bottom:10px}.download-hero h2{font-size:19px}.download-hero p{font-size:11px}.download-stats{grid-template-columns:1fr 1fr;gap:7px}.download-card{min-height:78px;padding:10px;gap:8px}.card-icon{width:30px;height:30px;font-size:17px}.card-label{font-size:9px}.download-card strong{font-size:17px}.download-card small{font-size:9px}.settings-modal{max-height:calc(100dvh - 24px);overflow:auto;padding:18px}.settings-head{margin-bottom:15px}.settings-modal label{margin-top:13px}} @media(max-width:700px){body{font-size:12px;padding-bottom:env(safe-area-inset-bottom)}header{height:auto;min-height:62px;padding:9px 12px;gap:9px;align-items:center}.brand-mark,.brand-mark svg{width:30px;height:30px}.brand-name{font-size:14px}.head-title{font-size:12px}.head-sub{font-size:10px}.actions{gap:4px}.actions button{height:34px;padding:0 8px}.settings-button{font-size:15px}.actions #rescan{font-size:0}.actions #rescan::after{content:'↻';font-size:16px}main{padding:10px 10px 24px}.nav{margin-bottom:10px;padding:4px;overflow-x:auto;scrollbar-width:none}.nav::-webkit-scrollbar{display:none}.nav button{flex:0 0 auto;white-space:nowrap;padding:8px 10px;font-size:11px}.toolbar{grid-template-columns:1fr;gap:7px;padding:8px}.toolbar input,.toolbar select{width:100%;height:38px}.stats{gap:6px}.stat{padding:6px 8px}.tablewrap{overflow-x:auto;border-radius:8px}table{min-width:860px}.dashboard-hero{margin-bottom:10px;padding:18px;border-radius:11px}.dashboard-hero h1{font-size:21px}.dashboard-hero p{font-size:11px;line-height:1.4}.overview-grid{grid-template-columns:1fr 1fr;gap:7px}.overview-card{min-height:84px;padding:12px 10px;gap:8px}.overview-icon{width:32px;height:32px;font-size:18px}.overview-card strong{font-size:18px}.overview-card span:not(.overview-icon){font-size:9px}.overview-card small{font-size:9px}.dashboard-columns{gap:8px}.system-panel,.service-panel{padding:14px;border-radius:10px}.system-meters{gap:12px}.panel-heading{margin-bottom:13px}.panel-heading h2{font-size:14px}.panel-heading p{font-size:10px}.disk-row{grid-template-columns:82px 1fr auto;gap:7px}.disk-values{font-size:10px}.download-hero{padding:18px;margin-bottom:10px}.download-hero h2{font-size:19px}.download-hero p{font-size:11px}.download-stats{grid-template-columns:1fr 1fr;gap:7px}.download-card{min-height:78px;padding:10px;gap:8px}.card-icon{width:30px;height:30px;font-size:17px}.card-label{font-size:9px}.download-card strong{font-size:17px}.download-card small{font-size:9px}.settings-modal{max-height:calc(100dvh - 24px);overflow:auto;padding:18px}.settings-head{margin-bottom:15px}.settings-modal label{margin-top:13px}}
@media(max-width:390px){.brand-name{display:none}.head-sub{display:none}.overview-grid{grid-template-columns:1fr}.dashboard-hero{padding:15px}.download-stats{grid-template-columns:1fr}} @media(max-width:390px){.brand-name{display:none}.head-sub{display:none}.overview-grid{grid-template-columns:1fr}.dashboard-hero{padding:15px}.download-stats{grid-template-columns:1fr}}
</style></head><body> </style><style>.scan-control{display:flex;align-items:center;gap:5px}.scan-control select{height:36px;padding:0 8px;font-size:11px}.scan-control button{white-space:nowrap}@media(max-width:700px){.scan-control select{max-width:108px;font-size:10px}.scan-control button{font-size:0}.scan-control button::after{content:'↻';font-size:16px}}</style></head><body>
<header><div class="brand"><div class="brand-mark" aria-label="Media Dashboard"><svg viewBox="0 0 48 48" role="img"><path d="M10 8h28a4 4 0 0 1 4 4v24a4 4 0 0 1-4 4H10a4 4 0 0 1-4-4V12a4 4 0 0 1 4-4Z" fill="#20252d" stroke="#60a9e6" stroke-width="2"/><path d="M14 17h20M14 24h13M14 31h20" stroke="#f4c430" stroke-width="3" stroke-linecap="round"/><circle cx="34" cy="24" r="5" fill="#159447" stroke="#c9ffe0" stroke-width="1.5"/></svg></div><div class="brand-name">MEDIA</div></div><div><div class="head-title" data-de="Language Dashboard" data-en="Language Dashboard">Language Dashboard</div><div class="head-sub" data-de="Audio- und Untertitelspuren aus den Mediendateien" data-en="Audio and subtitle tracks from media files">Audio- und Untertitelspuren aus den Mediendateien</div></div><div class="actions"><button class="lang-switch" id="languageToggle" title="Sprache / Language">DE</button><button class="settings-button" id="settingsOpen" title="Einstellungen"></button><button id="rescan" data-de="↻ Neu scannen" data-en="↻ Rescan">↻ Neu scannen</button></div></header> <header><div class="brand"><div class="brand-mark" aria-label="Media Dashboard"><svg viewBox="0 0 48 48" role="img"><path d="M10 8h28a4 4 0 0 1 4 4v24a4 4 0 0 1-4 4H10a4 4 0 0 1-4-4V12a4 4 0 0 1 4-4Z" fill="#20252d" stroke="#60a9e6" stroke-width="2"/><path d="M14 17h20M14 24h13M14 31h20" stroke="#f4c430" stroke-width="3" stroke-linecap="round"/><circle cx="34" cy="24" r="5" fill="#159447" stroke="#c9ffe0" stroke-width="1.5"/></svg></div><div class="brand-name">MEDIA</div></div><div><div class="head-title" data-de="Language Dashboard" data-en="Language Dashboard">Language Dashboard</div><div class="head-sub" data-de="Audio- und Untertitelspuren aus den Mediendateien" data-en="Audio and subtitle tracks from media files">Audio- und Untertitelspuren aus den Mediendateien</div></div><div class="actions"><button class="lang-switch" id="languageToggle" title="Sprache / Language">DE</button><button class="settings-button" id="settingsOpen" title="Einstellungen"></button><div class="scan-control"><select id="scanSource" aria-label="Scan-Quelle"><option value="all">Alles scannen</option><option value="radarr">Nur Radarr</option><option value="sonarr">Nur Sonarr</option></select><button id="rescan" data-de="↻ Neu scannen" data-en="↻ Rescan">↻ Neu scannen</button></div></div></header>
<main> <main>
{% if error %}<div class="banner"><strong>Fehler:</strong> {{ error }}</div>{% endif %} {% if error %}<div class="banner"><strong>Fehler:</strong> {{ error }}</div>{% endif %}
<nav class="nav"><button class="active" data-view="dashboard">▦ Dashboard</button><button data-view="radarr">RADARR · Filme</button>{% if sonarr_enabled %}<button data-view="sonarr">SONARR · Serien</button>{% endif %}<button data-view="missing">⚠ Ohne Deutsch <span class="nav-count">{{ missing_german_movies|length + (missing_german_series|map(attribute='episodes')|map('length')|sum) }}</span></button>{% if sab_enabled %}<button data-view="downloads">SABNZBD · Downloads</button>{% endif %}</nav> <nav class="nav"><button class="active" data-view="dashboard">▦ Dashboard</button><button data-view="radarr">RADARR · Filme</button>{% if sonarr_enabled %}<button data-view="sonarr">SONARR · Serien</button>{% endif %}<button data-view="missing">⚠ Ohne Deutsch <span class="nav-count">{{ missing_german_movies|length + (missing_german_series|map(attribute='episodes')|map('length')|sum) }}</span></button>{% if sab_enabled %}<button data-view="downloads">SABNZBD · Downloads</button>{% endif %}</nav>
@@ -35,10 +35,10 @@
const dashboardView=document.querySelector('#dashboard');if(dashboardView){dashboardView.classList.add('dashboard-view');const stats={{ dashboard_stats|tojson }};const insights=document.createElement('div');insights.className='dashboard-insights';insights.innerHTML=`<div class="insight-card"><span class="insight-icon">▣</span><div><strong>${stats.movie_files} von ${stats.movie_total} Filmen vorhanden</strong><small>${stats.movie_missing} fehlen · ${stats.movie_downloading} werden geladen</small></div></div><div class="insight-card"><span class="insight-icon">▤</span><div><strong>${stats.episode_files} von ${stats.episode_total} Episoden vorhanden</strong><small>${stats.episode_missing} fehlen in ${stats.series_total} Serien</small></div></div><div class="insight-card"><span class="insight-icon">◌</span><div><strong>Medienbestand geprüft</strong><small>Audio, Untertitel und Qualität werden überwacht</small></div></div><div class="insight-card"><span class="insight-icon">✓</span><div><strong>Live-Überwachung aktiv</strong><small>Systemwerte und Downloads aktualisieren sich automatisch</small></div></div>`;const columns=dashboardView.querySelector('.dashboard-columns');if(columns)dashboardView.insertBefore(insights,columns)} const dashboardView=document.querySelector('#dashboard');if(dashboardView){dashboardView.classList.add('dashboard-view');const stats={{ dashboard_stats|tojson }};const insights=document.createElement('div');insights.className='dashboard-insights';insights.innerHTML=`<div class="insight-card"><span class="insight-icon">▣</span><div><strong>${stats.movie_files} von ${stats.movie_total} Filmen vorhanden</strong><small>${stats.movie_missing} fehlen · ${stats.movie_downloading} werden geladen</small></div></div><div class="insight-card"><span class="insight-icon">▤</span><div><strong>${stats.episode_files} von ${stats.episode_total} Episoden vorhanden</strong><small>${stats.episode_missing} fehlen in ${stats.series_total} Serien</small></div></div><div class="insight-card"><span class="insight-icon">◌</span><div><strong>Medienbestand geprüft</strong><small>Audio, Untertitel und Qualität werden überwacht</small></div></div><div class="insight-card"><span class="insight-icon">✓</span><div><strong>Live-Überwachung aktiv</strong><small>Systemwerte und Downloads aktualisieren sich automatisch</small></div></div>`;const columns=dashboardView.querySelector('.dashboard-columns');if(columns)dashboardView.insertBefore(insights,columns)}
const rows=[...document.querySelectorAll('#movies tbody tr')], search=document.querySelector('#search'), audio=document.querySelector('#audioFilter'), subs=document.querySelector('#subFilter'), state=document.querySelector('#stateFilter'), count=document.querySelector('#visibleCount'); const rows=[...document.querySelectorAll('#movies tbody tr')], search=document.querySelector('#search'), audio=document.querySelector('#audioFilter'), subs=document.querySelector('#subFilter'), state=document.querySelector('#stateFilter'), count=document.querySelector('#visibleCount');
function apply(){const q=search.value.trim().toLowerCase();let n=0;rows.forEach(r=>{const ok=(!q||r.dataset.title.includes(q))&&(!audio.value||r.dataset.audio.includes(audio.value))&&(!subs.value||r.dataset.subs.includes(subs.value))&&(!state.value||r.dataset.state===state.value);r.classList.toggle('hidden',!ok);if(ok)n++});count.textContent=`${n} Filme`};[search,audio,subs,state].forEach(x=>x.addEventListener(x===search?'input':'change',apply));apply(); function apply(){const q=search.value.trim().toLowerCase();let n=0;rows.forEach(r=>{const ok=(!q||r.dataset.title.includes(q))&&(!audio.value||r.dataset.audio.includes(audio.value))&&(!subs.value||r.dataset.subs.includes(subs.value))&&(!state.value||r.dataset.state===state.value);r.classList.toggle('hidden',!ok);if(ok)n++});count.textContent=`${n} Filme`};[search,audio,subs,state].forEach(x=>x.addEventListener(x===search?'input':'change',apply));apply();
document.querySelectorAll('.nav button').forEach(b=>b.onclick=()=>{document.querySelectorAll('.nav button').forEach(x=>x.classList.remove('active'));document.querySelectorAll('.view').forEach(x=>x.classList.remove('active'));b.classList.add('active');document.querySelector('#'+b.dataset.view).classList.add('active');const rescan=document.querySelector('#rescan'),canRescan=['radarr','sonarr'].includes(b.dataset.view);rescan.dataset.source=canRescan?b.dataset.view:'radarr';rescan.disabled=!canRescan;rescan.title=canRescan?'Bibliothek neu scannen':'Für diesen Tab nicht verfügbar'}); document.querySelectorAll('.nav button').forEach(b=>b.onclick=()=>{document.querySelectorAll('.nav button').forEach(x=>x.classList.remove('active'));document.querySelectorAll('.view').forEach(x=>x.classList.remove('active'));b.classList.add('active');document.querySelector('#'+b.dataset.view).classList.add('active');const rescan=document.querySelector('#rescan'),sourceSelect=document.querySelector('#scanSource'),canRescan=['radarr','sonarr'].includes(b.dataset.view);if(canRescan)sourceSelect.value=b.dataset.view;rescan.disabled=false;rescan.title='Bibliothek neu scannen'});
function statusText(s){if(s.state==='running')return `Scan: ${s.done}/${s.total||'?'} Dateien`;if(s.state==='done')return 'Scan abgeschlossen';if(s.state==='error')return 'Scanfehler: '+s.error;return 'Scan wird vorbereitet …'} function statusText(s){if(s.state==='running')return `Scan: ${s.done}/${s.total||'?'} Dateien`;if(s.state==='done')return 'Scan abgeschlossen';if(s.state==='error')return 'Scanfehler: '+s.error;return 'Scan wird vorbereitet …'}
const seenRunning={radarr:false,sonarr:false};async function poll(){try{const data=await (await fetch('/api/scan-status')).json();document.querySelector('#radarrStatus').textContent=statusText(data.radarr);const sonarr=document.querySelector('#sonarrStatus');if(sonarr)sonarr.textContent=statusText(data.sonarr);['radarr','sonarr'].forEach(source=>{if(data[source].state==='running')seenRunning[source]=true;if(seenRunning[source]&&['done','error'].includes(data[source].state)){seenRunning[source]=false;setTimeout(()=>location.reload(),700)}})}catch(e){} };setInterval(poll,1200);poll(); const seenRunning={radarr:false,sonarr:false};async function poll(){try{const data=await (await fetch('/api/scan-status')).json();document.querySelector('#radarrStatus').textContent=statusText(data.radarr);const sonarr=document.querySelector('#sonarrStatus');if(sonarr)sonarr.textContent=statusText(data.sonarr);['radarr','sonarr'].forEach(source=>{if(data[source].state==='running')seenRunning[source]=true;if(seenRunning[source]&&['done','error'].includes(data[source].state)){seenRunning[source]=false;setTimeout(()=>location.reload(),700)}})}catch(e){} };setInterval(poll,1200);poll();
document.querySelector('#rescan').onclick=async()=>{const b=document.querySelector('#rescan');b.disabled=true;b.textContent='Scanne …';await fetch('/api/rescan?source='+(b.dataset.source||'radarr'),{method:'POST'});b.disabled=false;b.textContent='↻ Neu scannen';poll()}; document.querySelector('#rescan').onclick=async()=>{const b=document.querySelector('#rescan'),source=document.querySelector('#scanSource').value;b.disabled=true;b.textContent='Scanne …';await fetch('/api/rescan?source='+source,{method:'POST'});b.disabled=false;b.textContent='↻ Neu scannen';poll()};
</script> </script>
<script> <script>
const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)} const scanLabel=document.createElement('label');scanLabel.htmlFor='autoScan';scanLabel.textContent='Automatisches Scanning';const scanHelp=document.createElement('span');scanHelp.className='setting-help';scanHelp.textContent='Bibliotheken werden automatisch geprüft, solange das Dashboard geöffnet ist.';const scanSelect=document.createElement('select');scanSelect.id='autoScan';scanSelect.className='setting-select';[['0','Deaktiviert'],['1800000','Alle 30 Minuten'],['3600000','Jede Stunde'],['7200000','Alle 2 Stunden'],['10800000','Alle 3 Stunden'],['14400000','Alle 4 Stunden'],['18000000','Alle 5 Stunden']].forEach(([value,label])=>{const option=document.createElement('option');option.value=value;option.textContent=label;scanSelect.appendChild(option)});const settingsActions=document.querySelector('.settings-actions');if(settingsActions){settingsActions.before(scanLabel,scanHelp,scanSelect)}