dashcam-transfer/dashcam-transfer.py

#!/usr/bin/env python3
"""Transfer dashcam MP4 files from SD card to USB-SSD with date/camera structure."""

import argparse
import fcntl
import json
import logging
import os
import re
import shutil
import sys
import time
import urllib.request

SD_CARD = "/mnt/sdcard"
SOURCE_DIR = os.path.join(SD_CARD, "Novatek", "Movie")
SSD = "/mnt/ultratouch"
TARGET_BASE = os.path.join(SSD, "dashcam")
LOG_FILE = "/tmp/dashcam-transfer.log"
LOCK_FILE = "/tmp/dashcam-transfer.lock"

CAMERA_MAP = {"F": "front", "B": "back"}
FILE_PATTERN = re.compile(r"^(\d{8})\d{6}_\d+([FB])\.MP4$")

MAX_RETRIES = 3
RETRY_DELAY = 2
MAX_CONSECUTIVE_ERRORS = 5

MATRIX_API = "https://whatsed.eddydevink.nl/_matrix/client/v3"
MATRIX_TOKEN_FILE = "/home/aiaiaiaiagent/whatsed-new/secrets/sherpa-token"
MATRIX_ROOM_FILE = "/home/aiaiaiaiagent/whatsed-new/secrets/eddy-report-room"

UNREADABLE_REASON = (
    "SD-kaart onleesbaar (I/O-fout): lezer/kaart defect of niet netjes afgemeld. "
    "fsck.exfat en lezer/poort controleren. Geen bronbestanden verwijderd."
)


class TransferAborted(Exception):
    def __init__(self, ok, errors, consecutive, reason):
        self.ok = ok
        self.errors = errors
        self.consecutive = consecutive
        self.reason = reason
        super().__init__(reason)


def setup_logging():
    logging.basicConfig(
        filename=LOG_FILE,
        level=logging.INFO,
        format="[%(asctime)s] %(levelname)s %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
    )


def acquire_lock():
    lock_fd = open(LOCK_FILE, "w")
    try:
        fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
    except IOError:
        logging.warning("Lockfile actief — script al bezig")
        sys.exit(1)
    return lock_fd


def check_mount(path, name):
    if not os.path.ismount(path):
        logging.error("%s niet gemount: %s", name, path)
        sys.exit(1)


def get_mp4_files(src_dir):
    """Return matching (fname, date, camera) tuples, [] when absent, None on I/O error."""
    if not os.path.isdir(src_dir):
        return []
    try:
        entries = os.listdir(src_dir)
    except OSError as e:
        logging.error("SD-kaart directory niet leesbaar: %s", e)
        return None
    result = []
    for fname in entries:
        match = FILE_PATTERN.match(fname)
        if match:
            result.append((fname, match.group(1), match.group(2).upper()))
    return result


def check_source_readable(src_dir, files):
    """Smoke-test that the SD card actually returns data, before freeing space."""
    try:
        with os.scandir(src_dir):
            pass
        if files:
            sample = os.path.join(src_dir, files[0][0])
            with open(sample, "rb") as f:
                f.read(1)
    except OSError as e:
        logging.error("SD-kaart niet leesbaar (I/O-fout): %s", e)
        return False
    return True


def ensure_free_space(needed_bytes, target_base):
    stat = os.statvfs(target_base)
    free = stat.f_frsize * stat.f_bavail
    if free >= needed_bytes:
        return

    logging.warning(
        "Onvoldoende ruimte: %d bytes nodig, %d bytes vrij", needed_bytes, free
    )

    date_dirs = sorted(
        d
        for d in os.listdir(target_base)
        if re.match(r"^\d{8}$", d)
        and os.path.isdir(os.path.join(target_base, d))
    )

    freed = 0
    removed = []
    for date_name in date_dirs:
        if free + freed >= needed_bytes:
            break
        dir_path = os.path.join(target_base, date_name)
        dir_size = sum(
            os.path.getsize(os.path.join(dirpath, f))
            for dirpath, _, filenames in os.walk(dir_path)
            for f in filenames
        )
        shutil.rmtree(dir_path)
        freed += dir_size
        removed.append(date_name)
        logging.info("Ruimte gemaakt: %s verwijderd (%d bytes)", date_name, dir_size)

    if free + freed < needed_bytes:
        logging.error(
            "Nog steeds onvoldoende ruimte na verwijderen %d mappen: "
            "%d bytes nodig, %d bytes vrij",
            len(removed),
            needed_bytes,
            free + freed,
        )
        sys.exit(1)

    logging.info("Ruimte vrijgemaakt: %d bytes door %d mappen", freed, len(removed))


def copy_file(src, dst, attempts=MAX_RETRIES, delay=RETRY_DELAY):
    """Copy src → dst with retry. Raises the last OSError when all attempts fail."""
    last_error = None
    for attempt in range(1, attempts + 1):
        try:
            # copy (niet copy2): copy2 preserveert timestamps via os.utime, en exfat
            # mountuid=1000 + allow_utime=0022 weigeren dat voor niet-eigenaars —
            # daarmee faalde elke kopie toen de service als aiaiaiaiagent ging
            # draaien (EPERM). Op exfat is mode altijd 0777 en staat het
            # opnamemoment al in de bestandsnaam (YYYYMMDDHHMMSS).
            shutil.copy(src, dst)
            return
        except OSError as e:
            last_error = e
            logging.warning(
                "Kopieerpoging %d/%d mislukt voor %s: %s",
                attempt, attempts, os.path.basename(src), e,
            )
            if attempt < attempts:
                time.sleep(delay)
    raise last_error


def _safe_remove(path, name):
    try:
        os.remove(path)
    except OSError as e:
        logging.warning("Bron %s kon niet verwijderd worden (blijft op kaart): %s", name, e)


def transfer_files(files, source_dir, target_base):
    """Copy+verify+delete per file. Returns (ok, errors) or raises TransferAborted."""
    ok = 0
    errors = 0
    consecutive = 0
    last_error = None

    for fname, date_str, cam in files:
        cam_name = CAMERA_MAP[cam]
        target_dir = os.path.join(target_base, date_str, cam_name)
        src = os.path.join(source_dir, fname)
        dst = os.path.join(target_dir, fname)

        try:
            os.makedirs(target_dir, exist_ok=True)

            if os.path.exists(dst):
                src_size = os.path.getsize(src)
                dst_size = os.path.getsize(dst)
                if src_size == dst_size:
                    logging.info(
                        "Bestaat al en klopt: %s → %s/%s/", fname, date_str, cam_name
                    )
                    _safe_remove(src, fname)
                    ok += 1
                    consecutive = 0
                    continue
                logging.warning(
                    "Doel %s bestaat maar is anders (%d vs %d bytes) — overschrijf",
                    dst, dst_size, src_size,
                )
                os.remove(dst)

            copy_file(src, dst)
            src_size = os.path.getsize(src)
            dst_size = os.path.getsize(dst)
            if dst_size != src_size:
                raise OSError("Verificatie mislukt: %d vs %d bytes" % (src_size, dst_size))
        except OSError as e:
            errors += 1
            consecutive += 1
            last_error = e
            logging.error("Fout bij overzetten %s: %s", fname, e)
            if consecutive >= MAX_CONSECUTIVE_ERRORS:
                logging.error(
                    "Te veel opeenvolgende fouten (%d) — transfer gestopt", consecutive
                )
                raise TransferAborted(
                    ok, errors, consecutive,
                    f"SD-kaart onleesbaar (I/O-fout) — gestopt na {consecutive} "
                    f"opeenvolgende fouten. Laatste fout: {last_error}. "
                    f"Geen bronbestanden verwijderd.",
                )
            continue

        _safe_remove(src, fname)
        logging.info(
            "Overgezet: %s → %s/%s/ (%d bytes)", fname, date_str, cam_name, src_size
        )
        ok += 1
        consecutive = 0

    return ok, errors


def send_matrix_notification(ok, errors, total_bytes, aborted=False, reason=None):
    gb = total_bytes / (1024 ** 3)
    if aborted:
        body = (
            f"❌ Dashcam transfer GESTOPT: {reason} "
            f"({ok} ok, {errors} fout(en) tot dusver)"
        )
    elif errors == 0:
        body = (
            f"✅ Dashcam transfer voltooid: {ok} bestanden ({gb:.1f} GB) "
            f"overgezet, geen fouten"
        )
    else:
        body = (
            f"❌ Dashcam transfer voltooid: {ok} bestanden ({gb:.1f} GB) "
            f"overgezet, {errors} fout(en)"
        )
    try:
        with open(MATRIX_TOKEN_FILE) as f:
            token = f.read().strip()
        with open(MATRIX_ROOM_FILE) as f:
            room = f.read().strip()
        if not token or not room:
            raise OSError("token of room-ID onbekend (lege secret-file)")
        txn_id = "dashcam-%d" % int(time.time() * 1000)
        url = "%s/rooms/%s/send/m.room.message/%s" % (MATRIX_API, room, txn_id)
        payload = {"msgtype": "m.text", "body": body}
        data = json.dumps(payload).encode()
        req = urllib.request.Request(
            url, data=data, method="PUT",
            headers={"Content-Type": "application/json",
                     "Authorization": "Bearer " + token},
        )
        urllib.request.urlopen(req, timeout=10)
        logging.info("Matrix notificatie verstuurd")
    except Exception as e:
        logging.warning("Matrix notificatie mislukt: %s", e)


def parse_args():
    parser = argparse.ArgumentParser(description="Dashcam transfer tool")
    parser.add_argument(
        "--limit", "-n", type=int, default=0,
        help="Maximum aantal bestanden om over te zetten (0 = alle)"
    )
    return parser.parse_args()


def main():
    args = parse_args()
    setup_logging()
    lock_fd = acquire_lock()

    check_mount(SD_CARD, "SD-kaart")
    check_mount(SSD, "Ultratouch SSD")

    files = get_mp4_files(SOURCE_DIR)
    if files is None:
        send_matrix_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
        return
    if not files:
        logging.info("Geen MP4 bestanden gevonden in %s", SOURCE_DIR)
        return

    if args.limit > 0:
        files = files[:args.limit]

    if not check_source_readable(SOURCE_DIR, files):
        send_matrix_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
        return

    total_size = 0
    for fname, _, _ in files:
        try:
            total_size += os.path.getsize(os.path.join(SOURCE_DIR, fname))
        except OSError as e:
            logging.error("Grootte van %s niet leesbaar: %s", fname, e)
            send_matrix_notification(0, 0, 0, aborted=True, reason=UNREADABLE_REASON)
            return

    logging.info("Start transfer: %d bestanden, %d bytes", len(files), total_size)

    try:
        ensure_free_space(total_size, TARGET_BASE)
    except SystemExit:
        send_matrix_notification(
            0, 0, total_size, aborted=True,
            reason="Onvoldoende vrije ruimte op de SSD, ook na opschonen.",
        )
        raise

    try:
        ok, errors = transfer_files(files, SOURCE_DIR, TARGET_BASE)
    except TransferAborted as e:
        logging.info("Transfer gestopt: %d ok, %d fouten", e.ok, e.errors)
        send_matrix_notification(e.ok, e.errors, total_size, aborted=True, reason=e.reason)
        return

    logging.info("Transfer voltooid: %d ok, %d fouten", ok, errors)
    send_matrix_notification(ok, errors, total_size)


if __name__ == "__main__":
    main()