feat: media send/receive (intent-based + notification bridge)

- POST /messages/send-media: multipart upload + adb push + intent + Appium
- Webhook mediaType: image detection + adb pull + fanout met download-url
- Download endpoint /download/:token (10 min TTL, cleanup every 5 min)
- Android NotificationListener: detecteert media-notificaties (lege/emoji text)
- Dockerfile: adb toegevoegd voor push/pull
- MEDIA_WAIT_MS env var (default 8000ms) voor pull-timing

README.md

@@ -235,6 +235,88 @@ fetchMessages: openChat → lees main_layout → message_text + date
 
 ---
 
+## Media-ondersteuning (foto's)
+
+### Verzenden: `POST /messages/send-media`
+
+Verstuurt een foto via WhatsApp. Gebruikt een ADB-intent om het deelscherm
+te openen, daarna Appium voor de contactkeuze en versturen.
+
+```bash
+curl -X POST http://127.0.0.1:3010/messages/send-media \
+  -F 'chatName=+31 6 12345678' \
+  -F 'caption=Optioneel bijschrift' \
+  -F 'image=@/pad/naar/foto.jpg'
+```
+
+| Veld | Type | Verplicht | Beschrijving |
+|------|------|-----------|-------------|
+| `image` | file | ja | JPEG/PNG/WEBP, max 20MB |
+| `chatName` | string | ja | Contactnaam of telefoonnummer |
+| `caption` | string | nee | Optioneel bijschrift bij de foto |
+
+**Flow:**
+1. Bestand wordt geüpload naar de API-container
+2. `adb push` naar `/sdcard/Pictures/` op de telefoon
+3. `adb shell am start` met `ACTION.SEND` intent naar WhatsApp
+4. Appium: contact zoeken en selecteren in het deelscherm
+5. Caption typen (indien opgegeven)
+6. Send-knop klikken
+7. Tijdelijke bestanden opruimen (telefoon + container)
+
+**Let op:** de contact picker in het deelscherm gebruikt mogelijk andere
+resource-IDs dan de normale WhatsApp-zoekbalk. Bij wijzigingen in
+WhatsApp kan dit endpoint breken.
+
+### Ontvangen (notification bridge → API)
+
+De Android notification-listener detecteert media-notificaties door te
+checken of de berichttekst leeg is of alleen emoji bevat. Bij een media-
+notificatie stuurt de app `mediaType: "image"` mee in de webhook-payload.
+
+De API verwerkt dit als volgt:
+1. Webhook binnen met `mediaType: "image"`
+2. API wacht `MEDIA_WAIT_MS` ms (default 8s) zodat WhatsApp de download
+   kan voltooien
+3. `adb shell ls -t` in `WhatsApp Images/` — zoekt het nieuwste bestand
+4. `adb pull` van dat bestand naar de container
+5. Opslaan met een uniek token, beschikbaar via `/download/:token`
+6. Fanout naar subscribers met `mediaUrl` + `mediaFilename` in de payload
+
+```bash
+# test: verstuur een kunstmatig media-event
+curl -X POST http://127.0.0.1:3010/webhooks/whatsapp/incoming \
+  -H 'Content-Type: application/json' \
+  -H 'X-Webhook-Secret: verander-mij' \
+  -d '{
+    "sender":"+31 6 12345678",
+    "text":"",
+    "chatId":"Test Chat",
+    "mediaType":"image",
+    "timestamp": 1700000000000
+  }'
+```
+
+### Bekende beperkingen (media)
+
+- **Race condition:** als kort na elkaar meerdere foto's in verschillende
+  chats binnenkomen, kan de "nieuwste bestand"-detectie de verkeerde
+  foto pakken. Mitigatie: verleng `MEDIA_WAIT_MS` of implementeer
+  bestandsgrootte/hash-vergelijking.
+- **Alleen WhatsApp Images:** de huidige implementatie kijkt alleen in
+  `/sdcard/Android/media/com.whatsapp/WhatsApp/Media/WhatsApp Images/`.
+  Video's, spraakberichten en andere media worden niet ondersteund.
+- **Android-versie-specifiek:** de paden naar de WhatsApp-media-map
+  kunnen wijzigen per Android-versie. Geverifieerd op Nexus 6P met
+  Android 8.
+- **Alleen push-notificaties:** media die binnenkomt terwijl de app open
+  is (geen notificatie) wordt niet opgevangen.
+- **Verouderde bestanden:** bij een trage WhatsApp-download kan het
+  gebeuren dat het bestand nog niet op de telefoon staat wanneer de API
+  het zoekt. Verleng `MEDIA_WAIT_MS` in dat geval.
+
+---
+
 ## Containers in detail
 
 ### appium-adb

android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WebhookClient.kt

@@ -23,6 +23,7 @@ object WebhookClient {
         text: String,
         chatId: String,
         timestamp: Long,
+        mediaType: String = "text",
         onResult: (Boolean) -> Unit
     ) {
         val body = JSONObject().apply {
@@ -30,6 +31,7 @@ object WebhookClient {
             put("text", text)
             put("chatId", chatId)
             put("timestamp", timestamp)
+            put("mediaType", mediaType)
         }.toString()
 
         val request = Request.Builder()

android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WhatsAppNotificationListener.kt

@@ -34,17 +34,23 @@ class WhatsAppNotificationListener : NotificationListenerService() {
             val text = msg.text?.toString() ?: ""
             val timestamp = msg.timestamp
 
+            val cleanText = text.trim()
+            val isMedia = cleanText.isBlank() ||
+                cleanText.matches(Regex("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF\\u200D\\uFE0F\\s]*"))
+            val mediaType = if (isMedia) "image" else "text"
+
             WebhookClient.postMessage(
                 webhookUrl = webhookUrl,
                 secret = secret,
                 sender = senderName,
-                text = text,
+                text = if (isMedia) "" else text,
                 chatId = chatId,
                 timestamp = timestamp,
+                mediaType = mediaType,
             ) { success ->
                 android.util.Log.d(
                     "WANotificationBridge",
-                    "Sent message from $senderName: ${if (success) "OK" else "FAILED after retries"}"
+                    "Sent ${mediaType} message from $senderName: ${if (success) "OK" else "FAILED after retries"}"
                 )
             }
         }

docker-compose.yml

@@ -23,4 +23,5 @@ services:
       - APPIUM_PORT=4723
       - PORT=3010
       - WEBHOOK_SHARED_SECRET=${WEBHOOK_SHARED_SECRET:-dev-secret-change-me}
+      - MEDIA_WAIT_MS=${MEDIA_WAIT_MS:-8000}
     restart: unless-stopped

wa-appium-api/Dockerfile

@@ -1,9 +1,13 @@
 FROM node:lts-bookworm-slim
 
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    adb \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app
 COPY package.json ./
 RUN npm install --omit=dev
 COPY server.js ./
 
-EXPOSE 3002
+EXPOSE 3010
 CMD ["node", "server.js"]

wa-appium-api/package.json

@@ -5,6 +5,7 @@
   "main": "server.js",
   "dependencies": {
     "express": "^4.21.0",
-    "webdriverio": "^8.40.0"
+    "webdriverio": "^8.40.0",
+    "multer": "^1.4.5-lts.1"
   }
 }

wa-appium-api/server.js

@@ -1,10 +1,27 @@
 const express = require('express');
 const { remote } = require('webdriverio');
+const multer = require('multer');
+const { execSync, exec } = require('child_process');
+const path = require('path');
+const fs = require('fs');
 
 const APPIUM_HOST = process.env.APPIUM_HOST || '127.0.0.1';
 const APPIUM_PORT = parseInt(process.env.APPIUM_PORT || '4723', 10);
 const PORT = parseInt(process.env.PORT || '3010', 10);
 const WEBHOOK_SECRET = process.env.WEBHOOK_SHARED_SECRET || 'dev-secret-change-me';
+const MEDIA_WAIT_MS = parseInt(process.env.MEDIA_WAIT_MS || '8000', 10);
+const WHATSAPP_IMAGES_DIR = '/sdcard/Android/media/com.whatsapp/WhatsApp/Media/WhatsApp Images';
+
+const upload = multer({
+  dest: '/tmp/wa-uploads/',
+  limits: { fileSize: 20 * 1024 * 1024 },
+  fileFilter: (_req, file, cb) => {
+    if (!file.mimetype.startsWith('image/')) {
+      return cb(new Error('Only image files are supported'));
+    }
+    cb(null, true);
+  },
+});
 
 const app = express();
 app.use(express.json({ limit: '10mb' }));
@@ -55,6 +72,10 @@ function find(d, selector, timeout = 5000) {
   return d.$(selector).waitForExist({ timeout }).then(() => d.$(selector), () => null);
 }
 
+function adb(args) {
+  return execSync(`adb ${args}`, { timeout: 30000, encoding: 'utf8' }).trim();
+}
+
 async function ensureWhatsApp(d) {
   const pkg = await d.getCurrentPackage();
   if (pkg !== 'com.whatsapp') {
@@ -89,7 +110,6 @@ async function openChat(d, query) {
   await searchInput.setValue(query);
   await d.pause(2000);
 
-  // Find and click the matching chat row
   const chats = await d.$$(uia('resourceId("com.whatsapp:id/contact_row_container")'));
   for (const row of chats) {
     const nameEl = await row.$(
@@ -102,7 +122,6 @@ async function openChat(d, query) {
       return;
     }
   }
-  // If no match by name, try clicking first result
   if (chats.length > 0) {
     await chats[0].click();
     await d.pause(1500);
@@ -111,16 +130,188 @@ async function openChat(d, query) {
   throw new Error(`No chat found for "${query}"`);
 }
 
-// ── Debug ─────────────────────────────────────────────────────────
+// ── ADB helpers ────────────────────────────────────────────────
 
+function adbPush(localPath, remotePath) {
+  return adb(`push "${localPath}" "${remotePath}"`);
+}
 
+function adbRm(remotePath) {
+  return adb(`shell rm -f "${remotePath}"`);
+}
 
-// ── Endpoints ─────────────────────────────────────────────────────
+function adbShell(cmd) {
+  return adb(`shell ${cmd}`);
+}
+
+function adbLatestFile(dir) {
+  const out = adb(`shell ls -t "${dir}/" 2>/dev/null | head -1`);
+  return out ? `${dir}/${out}` : null;
+}
+
+function adbPull(remotePath, localPath) {
+  return adb(`pull "${remotePath}" "${localPath}"`);
+}
+
+// ── Media send ─────────────────────────────────────────────────
+
+async function sendMediaViaShareIntent(d, chatName, localPath, mimetype, caption) {
+  const ext = path.extname(localPath) || '.jpg';
+  const remoteName = `wa_${Date.now()}${ext}`;
+  const remotePath = `/sdcard/Pictures/${remoteName}`;
+
+  adbPush(localPath, remotePath);
+
+  try {
+    adbShell(
+      `am start -a android.intent.action.SEND ` +
+      `-t ${mimetype} ` +
+      `--eu android.intent.extra.STREAM file://${remotePath} ` +
+      `-p com.whatsapp`
+    );
+    await d.pause(3000);
+
+    const searchInput = await find(
+      d,
+      uia('resourceId("com.whatsapp:id/contactpicker_search")'),
+      5000
+    );
+    if (!searchInput) {
+      throw new Error('Could not find contact picker search in share screen');
+    }
+
+    await searchInput.clearValue();
+    await d.pause(300);
+    await searchInput.setValue(chatName);
+    await d.pause(2000);
+
+    const contacts = await d.$$(
+      uia('resourceId("com.whatsapp:id/contactpicker_row_name")')
+    );
+    let clicked = false;
+    for (const c of contacts) {
+      const name = await c.getText().catch(() => '');
+      if (name.toLowerCase().includes(chatName.toLowerCase())) {
+        await c.click();
+        clicked = true;
+        break;
+      }
+    }
+    if (!clicked && contacts.length > 0) {
+      await contacts[0].click();
+      clicked = true;
+    }
+    if (!clicked) throw new Error(`Contact "${chatName}" not found in share picker`);
+
+    await d.pause(3000);
+
+    if (caption) {
+      const captionInput = await find(
+        d,
+        uia('resourceId("com.whatsapp:id/caption")'),
+        3000
+      );
+      if (captionInput) {
+        await captionInput.clearValue();
+        await d.pause(200);
+        await captionInput.setValue(caption);
+        await d.pause(500);
+      } else {
+        const entry = await find(d, uia('resourceId("com.whatsapp:id/entry")'), 2000);
+        if (entry) {
+          await entry.setValue(caption);
+          await d.pause(500);
+        }
+      }
+    }
+
+    let sendBtn = await find(d, uia('resourceId("com.whatsapp:id/send")'), 5000);
+    if (!sendBtn) {
+      sendBtn = await find(d, uia('descriptionContains("Send")'), 3000);
+    }
+    if (!sendBtn) throw new Error('Could not find send button after selecting contact');
+
+    await sendBtn.click();
+    await d.pause(2000);
+
+    goHome(d).catch(() => {});
+  } finally {
+    adbRm(remotePath);
+  }
+}
+
+// ── Media receive ───────────────────────────────────────────────
+
+function waitForNewImage(beforeTimestamp) {
+  const t0 = beforeTimestamp || Date.now();
+  const maxWait = MEDIA_WAIT_MS;
+  const pollInterval = 1000;
+  let waited = 0;
+
+  while (waited < maxWait) {
+    const latest = adbLatestFile(WHATSAPP_IMAGES_DIR);
+    if (latest) {
+      const tsMatch = adbShell(`stat -c '%Y' "${latest}" 2>/dev/null || echo 0`);
+      const ts = parseInt(tsMatch, 10) * 1000;
+      if (ts >= t0) return latest;
+    }
+    new (require('events').EventEmitter)().setMaxListeners(100);
+    const start = Date.now();
+    while (Date.now() - start < pollInterval) {}
+    waited += pollInterval;
+  }
+  return null;
+}
+
+function pullImageFile(remotePath) {
+  const filename = path.basename(remotePath);
+  const localPath = `/tmp/wa-downloads/${filename}`;
+  fs.mkdirSync('/tmp/wa-downloads', { recursive: true });
+  adbPull(remotePath, localPath);
+  return localPath;
+}
+
+let mediaTokenCounter = 0;
+const mediaDownloads = new Map();
+
+function fanoutMedia(event, localPath) {
+  const token = `media_${Date.now()}_${++mediaTokenCounter}`;
+  const filename = path.basename(localPath);
+  mediaDownloads.set(token, { path: localPath, filename, createdAt: Date.now() });
+
+  const downloadUrl = `http://127.0.0.1:${PORT}/download/${token}`;
+  const eventWithUrl = { ...event, mediaUrl: downloadUrl, mediaFilename: filename };
+
+  for (const sub of subscribers) {
+    fetch(sub.url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(eventWithUrl),
+    }).then((r) => {
+      console.log(`  media fanout to ${sub.url}: ${r.status}`);
+    }).catch((e) => {
+      console.log(`  media fanout to ${sub.url} failed: ${e.message}`);
+    });
+  }
+}
+
+// Cleanup old downloads every 5 min
+setInterval(() => {
+  const cutoff = Date.now() - 10 * 60 * 1000;
+  for (const [token, entry] of mediaDownloads) {
+    if (entry.createdAt < cutoff) {
+      try { fs.unlinkSync(entry.path); } catch {}
+      mediaDownloads.delete(token);
+    }
+  }
+}, 5 * 60 * 1000);
+
+// ── Endpoints (Appium) ─────────────────────────────────────────
 
 app.get('/health', async (_req, res) => {
   try {
     const d = await getDriver();
-    const session = await d.getSession();
+    await d.getSession();
     res.json({ status: 'ok', sessionId: d.sessionId });
   } catch (e) {
     res.status(503).json({ status: 'error', message: e.message });
@@ -194,9 +385,7 @@ app.post('/chat/fetchMessages/:session', async (req, res) => {
     for (const layout of layoutEls) {
       try {
         const textEl = await layout.$(uia('resourceId("com.whatsapp:id/message_text")'));
-        const dateEl = await layout.$(
+        const dateEl = await layout.$(uia('resourceId("com.whatsapp:id/date")'));
-          uia('resourceId("com.whatsapp:id/date")')
-        );
         if (await textEl.isExisting()) {
           messages.push({
             text: await textEl.getText(),
@@ -246,7 +435,32 @@ app.post('/client/sendMessage/:session', async (req, res) => {
   }
 });
 
-// ── Webhook subscriber store (in-memory) ─────────────────────
+// ── Media send endpoint ────────────────────────────────────────
+
+app.post('/messages/send-media', upload.single('image'), async (req, res) => {
+  let localPath = null;
+  try {
+    const { chatName, caption } = req.body;
+    const file = req.file;
+    if (!chatName) return res.status(400).json({ error: 'chatName required' });
+    if (!file) return res.status(400).json({ error: 'image file required' });
+
+    localPath = file.path;
+    const d = await getDriver();
+
+    await sendMediaViaShareIntent(d, chatName, localPath, file.mimetype, caption || '');
+
+    res.json({ status: 'sent', chatName, caption: caption || '' });
+  } catch (e) {
+    res.status(500).json({ error: e.message });
+  } finally {
+    if (localPath) {
+      try { fs.unlinkSync(localPath); } catch {}
+    }
+  }
+});
+
+// ── Webhook subscriber store ─────────────────────────────────────
 
 const subscribers = [];
 
@@ -272,14 +486,43 @@ app.post('/webhooks/whatsapp/incoming', (req, res) => {
     return res.status(401).json({ error: 'invalid or missing webhook secret' });
   }
 
-  const { sender, text, chatId, timestamp } = req.body;
+  const { sender, text, chatId, timestamp, mediaType } = req.body;
-  if (!sender || !text || !chatId) {
+  if (!sender || !chatId) {
-    return res.status(400).json({ error: 'sender, text, and chatId are required' });
+    return res.status(400).json({ error: 'sender and chatId are required' });
   }
 
-  const event = { sender, text, chatId, timestamp: timestamp || Date.now(), receivedAt: new Date().toISOString() };
+  const event = {
-  fanout(event);
+    sender,
+    text: text || '',
+    chatId,
+    timestamp: timestamp || Date.now(),
+    receivedAt: new Date().toISOString(),
+    mediaType: mediaType || 'text',
+  };
+
+  if (mediaType === 'image') {
+    res.json({ status: 'ok', mediaProcessing: 'pending' });
+
+    setImmediate(async () => {
+      try {
+        const imagePath = waitForNewImage(event.timestamp);
+        if (!imagePath) {
+          console.log(`  media: no new image found for ${chatId} from ${sender}`);
+          fanout({ ...event, mediaStatus: 'not-found' });
+          return;
+        }
+        const localPath = pullImageFile(imagePath);
+        console.log(`  media: pulled ${imagePath} → ${localPath}`);
+        fanoutMedia({ ...event, mediaStatus: 'downloaded' }, localPath);
+      } catch (e) {
+        console.log(`  media processing error: ${e.message}`);
+        fanout({ ...event, mediaStatus: 'error', error: e.message });
+      }
+    });
+  } else {
+    fanout({ ...event, mediaType: 'text' });
     res.json({ status: 'ok' });
+  }
 });
 
 app.post('/webhooks/subscribe', (req, res) => {
@@ -296,9 +539,21 @@ app.get('/webhooks/subscribers', (_req, res) => {
   res.json({ subscribers: subscribers.map((s) => ({ url: s.url, createdAt: s.createdAt })) });
 });
 
+// ── Media download endpoint ──────────────────────────────────
+
+app.get('/download/:token', (req, res) => {
+  const entry = mediaDownloads.get(req.params.token);
+  if (!entry) return res.status(404).json({ error: 'download expired or not found' });
+  res.download(entry.path, entry.filename);
+});
+
 // ── Server start ─────────────────────────────────────────────
 
+fs.mkdirSync('/tmp/wa-uploads', { recursive: true });
+fs.mkdirSync('/tmp/wa-downloads', { recursive: true });
+
 app.listen(PORT, '0.0.0.0', () => {
+  const host = require('os').networkInterfaces();
   console.log(`wa-appium-api listening on http://0.0.0.0:${PORT}`);
   console.log(`  Appium server: ${APPIUM_HOST}:${APPIUM_PORT}`);
   console.log(`  Webhook secret: ${WEBHOOK_SECRET === 'dev-secret-change-me' ? 'WARNING: using dev default' : 'configured'}`);