nessi/language-dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add RAID/storage error resilience with stale data fallback and parallel SMART health checks

Browse Source 
Add datastore_cache raid/raid_at fields for RAID status caching with 60-second fallback window. Extract smart_status() function from raid_status() to enable parallel execution. Add ThreadPoolExecutor submission for SMART checks across all RAID disks with concurrent futures. Update SMART logging to check "PASSED" status explicitly instead of regex match presence. Add stale/warning fields to datastore_snapshot() when fresh
This commit is contained in:
nessi
2026-08-16 12:34:26 +02:00
parent cc7d719465
commit 44fbfaded4
1 changed files with 40 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.py
+40 -18
View File
@@ -66,7 +66,7 @@ previous_cpu = None
service_status_lock = threading.Lock()
service_check_executor = ThreadPoolExecutor(max_workers=16)
datastore_cache_lock = threading.Lock()
datastore_cache = {"stored_at": 0.0, "snapshot": None}
datastore_cache = {"stored_at": 0.0, "snapshot": None, "raid": None, "raid_at": 0.0}
raid_log_state = None
hardware_log_once = set()
@@ -322,6 +322,20 @@ def format_remaining_time(value):
parts.append(f"{minutes} Min.")
return " ".join(parts) or "< 1 Min."
def smart_status(device):
if not SMART_ENABLED:
return "Nicht aktiviert"
if not shutil.which("smartctl"):
return "smartctl nicht installiert"
smart_device = device.replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1)
smart_output, smart_error = _read_command(["smartctl", "-H", smart_device], timeout=6, warn_on_nonzero=False)
if not re.search(r"(?:overall-health|Health Status|SMART support is)", smart_output, re.IGNORECASE):
sat_output, sat_error = _read_command(["smartctl", "-H", "-d", "sat", smart_device], timeout=6, warn_on_nonzero=False)
if sat_output:
smart_output, smart_error = sat_output, sat_error
health = re.search(r"(?:SMART overall-health self-assessment test result|SMART Health Status):\s*(.+)", smart_output, re.IGNORECASE)
return health.group(1).strip() if health else ("Fehler: " + smart_error.strip()[:80] if smart_error.strip() else "Nicht unterstützt")
def raid_status():
global raid_log_state
mdstat, mdstat_error = "", ""
@@ -375,6 +389,9 @@ def raid_status():
block_devices = {item.get("path") or f"/dev/{item.get('name')}": item for item in _flatten_lsblk(json.loads(lsblk_output).get("blockdevices", []))}
except (ValueError, AttributeError):
block_devices = {}
smart_futures = {}
if SMART_ENABLED and shutil.which("smartctl"):
smart_futures = {disk["device"]: service_check_executor.submit(smart_status, disk["device"]) for disk in devices}
for disk in devices:
info = block_devices.get(disk["device"], {})
parent = re.sub(r"(p)?\d+$", "", disk["device"])
@@ -393,25 +410,15 @@ def raid_status():
if not serial and serial_log_key not in hardware_log_once:
log.warning("Keine Seriennummer für %s gefunden; Host-Controller stellt möglicherweise keine bereit", disk["device"])
hardware_log_once.add(serial_log_key)
if SMART_ENABLED and shutil.which("smartctl"):
smart_device = disk["device"].replace("/dev/", f"{HOST_DEVICE_PATH.rstrip('/')}/", 1)
smart_output, smart_error = _read_command(["smartctl", "-H", smart_device], timeout=6, warn_on_nonzero=False)
if not re.search(r"(?:overall-health|Health Status|SMART support is)", smart_output, re.IGNORECASE):
sat_output, sat_error = _read_command(["smartctl", "-H", "-d", "sat", smart_device], timeout=6, warn_on_nonzero=False)
if sat_output:
smart_output, smart_error = sat_output, sat_error
health = re.search(r"(?:SMART overall-health self-assessment test result|SMART Health Status):\s*(.+)", smart_output, re.IGNORECASE)
disk["smart"] = health.group(1).strip() if health else ("Fehler: " + smart_error.strip()[:80] if smart_error.strip() else "Nicht unterstützt")
if SMART_ENABLED:
disk["smart"] = smart_futures[disk["device"]].result() if disk["device"] in smart_futures else smart_status(disk["device"])
smart_log_key = "smart:" + disk["device"]
if health and "smart-info:" + disk["device"] not in hardware_log_once:
log.info("SMART-Status %s: %s", disk["device"], disk["smart"])
if disk["smart"] == "PASSED" and "smart-info:" + disk["device"] not in hardware_log_once:
log.info("SMART-Status %s: PASSED", disk["device"])
hardware_log_once.add("smart-info:" + disk["device"])
if not health and smart_log_key not in hardware_log_once:
elif disk["smart"] != "PASSED" and smart_log_key not in hardware_log_once:
log.warning("SMART-Status für %s nicht verfügbar: %s", disk["device"], disk["smart"])
hardware_log_once.add(smart_log_key)
elif SMART_ENABLED and "smartctl" not in hardware_log_once:
log.error("smartctl ist im Container nicht verfügbar; SMART kann nicht geprüft werden")
hardware_log_once.add("smartctl")
missing = members_match and members_match.group(1) != members_match.group(2)
failed = detail.get("failed_devices", 0) > 0 or any(disk["status"] == "failed" for disk in devices)
rebuilding = bool(progress_match)
@@ -445,10 +452,25 @@ def datastore_snapshot():
now = time.time()
with datastore_cache_lock:
if datastore_cache["snapshot"] is None or now - datastore_cache["stored_at"] >= max(30, STORAGE_CACHE_SECONDS):
datastore_cache["snapshot"] = scan_datastore_storage()
fresh_storage = scan_datastore_storage()
if fresh_storage.get("error") and datastore_cache["snapshot"]:
datastore_cache["snapshot"] = dict(datastore_cache["snapshot"])
datastore_cache["snapshot"]["stale"] = True
datastore_cache["snapshot"]["warning"] = fresh_storage["error"]
else:
datastore_cache["snapshot"] = fresh_storage
datastore_cache["stored_at"] = now
storage = datastore_cache["snapshot"]
return {"storage": storage, "raid": raid_status(), "cache_seconds": STORAGE_CACHE_SECONDS}
current_raid = raid_status()
with datastore_cache_lock:
if current_raid.get("status") == "failed" and current_raid.get("error") and datastore_cache.get("raid") and now - datastore_cache.get("raid_at", 0) < 60:
current_raid = dict(datastore_cache["raid"])
current_raid["stale"] = True
current_raid["warning"] = "Vorübergehend keine aktuelle RAID-Antwort; letzter gültiger Stand wird angezeigt."
elif current_raid.get("status") != "failed":
datastore_cache["raid"] = dict(current_raid)
datastore_cache["raid_at"] = now
return {"storage": storage, "raid": current_raid, "cache_seconds": STORAGE_CACHE_SECONDS}
def cpu_usage_percent():
global previous_cpu