feat: harden dashcam transfer tegen flakke SD-lezer + stale-mount cleanup
- dashcam-transfer.py: log echte OSError (shutil.copy2 i.p.v. cp), pre-flight leesbaarheidscheck voor ruimte-wissen, retry (3x) + early-abort (5x), guards op bestandssysteem-calls, WhatsApp GESTOPT bij defecte kaart - dashcam-transfer-mount.sh: skip als service al draait, read-only fsck.exfat health check, %k device-naam doorgeven - dashcam-transfer-cleanup.sh: nieuw - ruimt stale mount op bij udev REMOVE - udev/99-dashcam-trigger.rules: ADD + REMOVE regels met %k - tests: 20 python + 7 mount + 12 cleanup checks - README: architectuur, robuustheid, deploy-instructies
This commit is contained in:
Eddy de Vink
parent
8ef7078ba4
commit
8cc04ea409
9 changed files with 779 additions and 76 deletions
|
|
@ -8,8 +8,8 @@ import logging
|
|||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
SD_CARD = "/mnt/sdcard"
|
||||
|
|
@ -22,6 +22,27 @@ LOCK_FILE = "/tmp/dashcam-transfer.lock"
|
|||
CAMERA_MAP = {"F": "front", "B": "back"}
|
||||
FILE_PATTERN = re.compile(r"^(\d{8})\d{6}_\d+([FB])\.MP4$")
|
||||
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 2
|
||||
MAX_CONSECUTIVE_ERRORS = 5
|
||||
|
||||
WHATSAPP_API = "http://localhost:3001/client/sendMessage/EdGpt" # localhost = nextcloud server, niet Mac Mini!
|
||||
WHATSAPP_CHAT = "31620248011@c.us"
|
||||
|
||||
UNREADABLE_REASON = (
|
||||
"SD-kaart onleesbaar (I/O-fout): lezer/kaart defect of niet netjes afgemeld. "
|
||||
"fsck.exfat en lezer/poort controleren. Geen bronbestanden verwijderd."
|
||||
)
|
||||
|
||||
|
||||
class TransferAborted(Exception):
|
||||
def __init__(self, ok, errors, consecutive, reason):
|
||||
self.ok = ok
|
||||
self.errors = errors
|
||||
self.consecutive = consecutive
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
def setup_logging():
|
||||
logging.basicConfig(
|
||||
|
|
@ -49,16 +70,37 @@ def check_mount(path, name):
|
|||
|
||||
|
||||
def get_mp4_files(src_dir):
|
||||
"""Return matching (fname, date, camera) tuples, [] when absent, None on I/O error."""
|
||||
if not os.path.isdir(src_dir):
|
||||
return []
|
||||
try:
|
||||
entries = os.listdir(src_dir)
|
||||
except OSError as e:
|
||||
logging.error("SD-kaart directory niet leesbaar: %s", e)
|
||||
return None
|
||||
result = []
|
||||
for fname in os.listdir(src_dir):
|
||||
for fname in entries:
|
||||
match = FILE_PATTERN.match(fname)
|
||||
if match:
|
||||
result.append((fname, match.group(1), match.group(2).upper()))
|
||||
return result
|
||||
|
||||
|
||||
def check_source_readable(src_dir, files):
|
||||
"""Smoke-test that the SD card actually returns data, before freeing space."""
|
||||
try:
|
||||
with os.scandir(src_dir):
|
||||
pass
|
||||
if files:
|
||||
sample = os.path.join(src_dir, files[0][0])
|
||||
with open(sample, "rb") as f:
|
||||
f.read(1)
|
||||
except OSError as e:
|
||||
logging.error("SD-kaart niet leesbaar (I/O-fout): %s", e)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def ensure_free_space(needed_bytes, target_base):
|
||||
stat = os.statvfs(target_base)
|
||||
free = stat.f_frsize * stat.f_bavail
|
||||
|
|
@ -105,19 +147,113 @@ def ensure_free_space(needed_bytes, target_base):
|
|||
logging.info("Ruimte vrijgemaakt: %d bytes door %d mappen", freed, len(removed))
|
||||
|
||||
|
||||
WHATSAPP_API = "http://localhost:3001/client/sendMessage/EdGpt" # localhost = nextcloud server, niet Mac Mini!
|
||||
WHATSAPP_CHAT = "31620248011@c.us"
|
||||
def copy_file(src, dst, attempts=MAX_RETRIES, delay=RETRY_DELAY):
|
||||
"""Copy src → dst with retry. Raises the last OSError when all attempts fail."""
|
||||
last_error = None
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
shutil.copy2(src, dst)
|
||||
return
|
||||
except OSError as e:
|
||||
last_error = e
|
||||
logging.warning(
|
||||
"Kopieerpoging %d/%d mislukt voor %s: %s",
|
||||
attempt, attempts, os.path.basename(src), e,
|
||||
)
|
||||
if attempt < attempts:
|
||||
time.sleep(delay)
|
||||
raise last_error
|
||||
|
||||
|
||||
def send_whatsapp_notification(ok, errors, total_bytes):
|
||||
if errors == 0:
|
||||
prefix = "✅"
|
||||
status = "geen fouten"
|
||||
else:
|
||||
prefix = "❌"
|
||||
status = f"{errors} fout(en)"
|
||||
def _safe_remove(path, name):
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError as e:
|
||||
logging.warning("Bron %s kon niet verwijderd worden (blijft op kaart): %s", name, e)
|
||||
|
||||
|
||||
def transfer_files(files, source_dir, target_base):
|
||||
"""Copy+verify+delete per file. Returns (ok, errors) or raises TransferAborted."""
|
||||
ok = 0
|
||||
errors = 0
|
||||
consecutive = 0
|
||||
last_error = None
|
||||
|
||||
for fname, date_str, cam in files:
|
||||
cam_name = CAMERA_MAP[cam]
|
||||
target_dir = os.path.join(target_base, date_str, cam_name)
|
||||
src = os.path.join(source_dir, fname)
|
||||
dst = os.path.join(target_dir, fname)
|
||||
|
||||
try:
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
if os.path.exists(dst):
|
||||
src_size = os.path.getsize(src)
|
||||
dst_size = os.path.getsize(dst)
|
||||
if src_size == dst_size:
|
||||
logging.info(
|
||||
"Bestaat al en klopt: %s → %s/%s/", fname, date_str, cam_name
|
||||
)
|
||||
_safe_remove(src, fname)
|
||||
ok += 1
|
||||
consecutive = 0
|
||||
continue
|
||||
logging.warning(
|
||||
"Doel %s bestaat maar is anders (%d vs %d bytes) — overschrijf",
|
||||
dst, dst_size, src_size,
|
||||
)
|
||||
os.remove(dst)
|
||||
|
||||
copy_file(src, dst)
|
||||
src_size = os.path.getsize(src)
|
||||
dst_size = os.path.getsize(dst)
|
||||
if dst_size != src_size:
|
||||
raise OSError("Verificatie mislukt: %d vs %d bytes" % (src_size, dst_size))
|
||||
except OSError as e:
|
||||
errors += 1
|
||||
consecutive += 1
|
||||
last_error = e
|
||||
logging.error("Fout bij overzetten %s: %s", fname, e)
|
||||
if consecutive >= MAX_CONSECUTIVE_ERRORS:
|
||||
logging.error(
|
||||
"Te veel opeenvolgende fouten (%d) — transfer gestopt", consecutive
|
||||
)
|
||||
raise TransferAborted(
|
||||
ok, errors, consecutive,
|
||||
f"SD-kaart onleesbaar (I/O-fout) — gestopt na {consecutive} "
|
||||
f"opeenvolgende fouten. Laatste fout: {last_error}. "
|
||||
f"Geen bronbestanden verwijderd.",
|
||||
)
|
||||
continue
|
||||
|
||||
_safe_remove(src, fname)
|
||||
logging.info(
|
||||
"Overgezet: %s → %s/%s/ (%d bytes)", fname, date_str, cam_name, src_size
|
||||
)
|
||||
ok += 1
|
||||
consecutive = 0
|
||||
|
||||
return ok, errors
|
||||
|
||||
|
||||
def send_whatsapp_notification(ok, errors, total_bytes, aborted=False, reason=None):
|
||||
gb = total_bytes / (1024 ** 3)
|
||||
body = f"{prefix} Dashcam transfer voltooid: {ok} bestanden ({gb:.1f} GB) overgezet, {status}"
|
||||
if aborted:
|
||||
body = (
|
||||
f"❌ Dashcam transfer GESTOPT: {reason} "
|
||||
f"({ok} ok, {errors} fout(en) tot dusver)"
|
||||
)
|
||||
elif errors == 0:
|
||||
body = (
|
||||
f"✅ Dashcam transfer voltooid: {ok} bestanden ({gb:.1f} GB) "
|
||||
f"overgezet, geen fouten"
|
||||
)
|
||||
else:
|
||||
body = (
|
||||
f"❌ Dashcam transfer voltooid: {ok} bestanden ({gb:.1f} GB) "
|
||||
f"overgezet, {errors} fout(en)"
|
||||
)
|
||||
payload = {"chatId": WHATSAPP_CHAT, "content": body, "contentType": "string"}
|
||||
try:
|
||||
data = json.dumps(payload).encode()
|
||||
|
|
@ -148,6 +284,9 @@ def main():
|
|||
check_mount(SSD, "Ultratouch SSD")
|
||||
|
||||
files = get_mp4_files(SOURCE_DIR)
|
||||
if files is None:
|
||||
send_whatsapp_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
|
||||
return
|
||||
if not files:
|
||||
logging.info("Geen MP4 bestanden gevonden in %s", SOURCE_DIR)
|
||||
return
|
||||
|
|
@ -155,63 +294,36 @@ def main():
|
|||
if args.limit > 0:
|
||||
files = files[:args.limit]
|
||||
|
||||
total_size = sum(
|
||||
os.path.getsize(os.path.join(SOURCE_DIR, fname)) for fname, _, _ in files
|
||||
)
|
||||
logging.info(
|
||||
"Start transfer: %d bestanden, %d bytes", len(files), total_size
|
||||
)
|
||||
|
||||
ensure_free_space(total_size, TARGET_BASE)
|
||||
|
||||
ok = 0
|
||||
errors = 0
|
||||
for fname, date_str, cam in files:
|
||||
target_dir = os.path.join(TARGET_BASE, date_str, CAMERA_MAP[cam])
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
src = os.path.join(SOURCE_DIR, fname)
|
||||
dst = os.path.join(target_dir, fname)
|
||||
|
||||
if os.path.exists(dst):
|
||||
src_size = os.path.getsize(src)
|
||||
dst_size = os.path.getsize(dst)
|
||||
if src_size == dst_size:
|
||||
logging.info(
|
||||
"Bestaat al en klopt: %s → %s/%s/", fname, date_str, CAMERA_MAP[cam]
|
||||
)
|
||||
os.remove(src)
|
||||
ok += 1
|
||||
continue
|
||||
else:
|
||||
logging.warning(
|
||||
"Doel %s bestaat maar is anders (%d vs %d bytes) — overschrijf",
|
||||
dst, dst_size, src_size,
|
||||
)
|
||||
os.remove(dst)
|
||||
if not check_source_readable(SOURCE_DIR, files):
|
||||
send_whatsapp_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
|
||||
return
|
||||
|
||||
total_size = 0
|
||||
for fname, _, _ in files:
|
||||
try:
|
||||
subprocess.run(
|
||||
["cp", src, dst], check=True, capture_output=True, timeout=600
|
||||
)
|
||||
except (subprocess.CalledProcessError, OSError) as e:
|
||||
logging.error("Fout bij kopiëren %s: %s", fname, e)
|
||||
errors += 1
|
||||
continue
|
||||
total_size += os.path.getsize(os.path.join(SOURCE_DIR, fname))
|
||||
except OSError as e:
|
||||
logging.error("Grootte van %s niet leesbaar: %s", fname, e)
|
||||
send_whatsapp_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
|
||||
return
|
||||
|
||||
dst_size = os.path.getsize(dst)
|
||||
src_size = os.path.getsize(src)
|
||||
if dst_size != src_size:
|
||||
logging.error(
|
||||
"Verificatie mislukt %s: %d vs %d bytes", fname, src_size, dst_size
|
||||
)
|
||||
errors += 1
|
||||
continue
|
||||
logging.info("Start transfer: %d bestanden, %d bytes", len(files), total_size)
|
||||
|
||||
os.remove(src)
|
||||
logging.info(
|
||||
"Overgezet: %s → %s/%s/ (%d bytes)", fname, date_str, CAMERA_MAP[cam], src_size
|
||||
try:
|
||||
ensure_free_space(total_size, TARGET_BASE)
|
||||
except SystemExit:
|
||||
send_whatsapp_notification(
|
||||
0, 0, total_size, aborted=True,
|
||||
reason="Onvoldoende vrije ruimte op de SSD, ook na opschonen.",
|
||||
)
|
||||
ok += 1
|
||||
raise
|
||||
|
||||
try:
|
||||
ok, errors = transfer_files(files, SOURCE_DIR, TARGET_BASE)
|
||||
except TransferAborted as e:
|
||||
logging.info("Transfer gestopt: %d ok, %d fouten", e.ok, e.errors)
|
||||
send_whatsapp_notification(e.ok, e.errors, total_size, aborted=True, reason=e.reason)
|
||||
return
|
||||
|
||||
logging.info("Transfer voltooid: %d ok, %d fouten", ok, errors)
|
||||
send_whatsapp_notification(ok, errors, total_size)
|
||||
|
|
|
|||
Loading…
Reference in a new issue