diff --git a/README.md b/README.md index 3aa8f0d..7a32e7f 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,88 @@ fetchMessages: openChat → lees main_layout → message_text + date --- +## Media-ondersteuning (foto's) + +### Verzenden: `POST /messages/send-media` + +Verstuurt een foto via WhatsApp. Gebruikt een ADB-intent om het deelscherm +te openen, daarna Appium voor de contactkeuze en versturen. + +```bash +curl -X POST http://127.0.0.1:3010/messages/send-media \ + -F 'chatName=+31 6 12345678' \ + -F 'caption=Optioneel bijschrift' \ + -F 'image=@/pad/naar/foto.jpg' +``` + +| Veld | Type | Verplicht | Beschrijving | +|------|------|-----------|-------------| +| `image` | file | ja | JPEG/PNG/WEBP, max 20MB | +| `chatName` | string | ja | Contactnaam of telefoonnummer | +| `caption` | string | nee | Optioneel bijschrift bij de foto | + +**Flow:** +1. Bestand wordt geüpload naar de API-container +2. `adb push` naar `/sdcard/Pictures/` op de telefoon +3. `adb shell am start` met `ACTION.SEND` intent naar WhatsApp +4. Appium: contact zoeken en selecteren in het deelscherm +5. Caption typen (indien opgegeven) +6. Send-knop klikken +7. Tijdelijke bestanden opruimen (telefoon + container) + +**Let op:** de contact picker in het deelscherm gebruikt mogelijk andere +resource-IDs dan de normale WhatsApp-zoekbalk. Bij wijzigingen in +WhatsApp kan dit endpoint breken. + +### Ontvangen (notification bridge → API) + +De Android notification-listener detecteert media-notificaties door te +checken of de berichttekst leeg is of alleen emoji bevat. Bij een media- +notificatie stuurt de app `mediaType: "image"` mee in de webhook-payload. + +De API verwerkt dit als volgt: +1. Webhook binnen met `mediaType: "image"` +2. API wacht `MEDIA_WAIT_MS` ms (default 8s) zodat WhatsApp de download + kan voltooien +3. `adb shell ls -t` in `WhatsApp Images/` — zoekt het nieuwste bestand +4. `adb pull` van dat bestand naar de container +5. Opslaan met een uniek token, beschikbaar via `/download/:token` +6. Fanout naar subscribers met `mediaUrl` + `mediaFilename` in de payload + +```bash +# test: verstuur een kunstmatig media-event +curl -X POST http://127.0.0.1:3010/webhooks/whatsapp/incoming \ + -H 'Content-Type: application/json' \ + -H 'X-Webhook-Secret: verander-mij' \ + -d '{ + "sender":"+31 6 12345678", + "text":"", + "chatId":"Test Chat", + "mediaType":"image", + "timestamp": 1700000000000 + }' +``` + +### Bekende beperkingen (media) + +- **Race condition:** als kort na elkaar meerdere foto's in verschillende + chats binnenkomen, kan de "nieuwste bestand"-detectie de verkeerde + foto pakken. Mitigatie: verleng `MEDIA_WAIT_MS` of implementeer + bestandsgrootte/hash-vergelijking. +- **Alleen WhatsApp Images:** de huidige implementatie kijkt alleen in + `/sdcard/Android/media/com.whatsapp/WhatsApp/Media/WhatsApp Images/`. + Video's, spraakberichten en andere media worden niet ondersteund. +- **Android-versie-specifiek:** de paden naar de WhatsApp-media-map + kunnen wijzigen per Android-versie. Geverifieerd op Nexus 6P met + Android 8. +- **Alleen push-notificaties:** media die binnenkomt terwijl de app open + is (geen notificatie) wordt niet opgevangen. +- **Verouderde bestanden:** bij een trage WhatsApp-download kan het + gebeuren dat het bestand nog niet op de telefoon staat wanneer de API + het zoekt. Verleng `MEDIA_WAIT_MS` in dat geval. + +--- + ## Containers in detail ### appium-adb diff --git a/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WebhookClient.kt b/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WebhookClient.kt index 83321ce..3efff79 100644 --- a/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WebhookClient.kt +++ b/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WebhookClient.kt @@ -23,6 +23,7 @@ object WebhookClient { text: String, chatId: String, timestamp: Long, + mediaType: String = "text", onResult: (Boolean) -> Unit ) { val body = JSONObject().apply { @@ -30,6 +31,7 @@ object WebhookClient { put("text", text) put("chatId", chatId) put("timestamp", timestamp) + put("mediaType", mediaType) }.toString() val request = Request.Builder() diff --git a/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WhatsAppNotificationListener.kt b/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WhatsAppNotificationListener.kt index 0682088..6a1ffdd 100644 --- a/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WhatsAppNotificationListener.kt +++ b/android-notification-listener/app/src/main/java/com/eddydevink/whatsapplistener/WhatsAppNotificationListener.kt @@ -34,17 +34,23 @@ class WhatsAppNotificationListener : NotificationListenerService() { val text = msg.text?.toString() ?: "" val timestamp = msg.timestamp + val cleanText = text.trim() + val isMedia = cleanText.isBlank() || + cleanText.matches(Regex("[\\uD83C-\\uDBFF\\uDC00-\\uDFFF\\u200D\\uFE0F\\s]*")) + val mediaType = if (isMedia) "image" else "text" + WebhookClient.postMessage( webhookUrl = webhookUrl, secret = secret, sender = senderName, - text = text, + text = if (isMedia) "" else text, chatId = chatId, timestamp = timestamp, + mediaType = mediaType, ) { success -> android.util.Log.d( "WANotificationBridge", - "Sent message from $senderName: ${if (success) "OK" else "FAILED after retries"}" + "Sent ${mediaType} message from $senderName: ${if (success) "OK" else "FAILED after retries"}" ) } } diff --git a/docker-compose.yml b/docker-compose.yml index 9725dde..418918a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,4 +23,5 @@ services: - APPIUM_PORT=4723 - PORT=3010 - WEBHOOK_SHARED_SECRET=${WEBHOOK_SHARED_SECRET:-dev-secret-change-me} + - MEDIA_WAIT_MS=${MEDIA_WAIT_MS:-8000} restart: unless-stopped diff --git a/wa-appium-api/Dockerfile b/wa-appium-api/Dockerfile index 0f1b65a..fb10703 100644 --- a/wa-appium-api/Dockerfile +++ b/wa-appium-api/Dockerfile @@ -1,9 +1,13 @@ FROM node:lts-bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + adb \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app COPY package.json ./ RUN npm install --omit=dev COPY server.js ./ -EXPOSE 3002 +EXPOSE 3010 CMD ["node", "server.js"] diff --git a/wa-appium-api/package.json b/wa-appium-api/package.json index aefaf40..1ef4e6f 100644 --- a/wa-appium-api/package.json +++ b/wa-appium-api/package.json @@ -5,6 +5,7 @@ "main": "server.js", "dependencies": { "express": "^4.21.0", - "webdriverio": "^8.40.0" + "webdriverio": "^8.40.0", + "multer": "^1.4.5-lts.1" } } diff --git a/wa-appium-api/server.js b/wa-appium-api/server.js index 5153f7d..62466ff 100644 --- a/wa-appium-api/server.js +++ b/wa-appium-api/server.js @@ -1,10 +1,27 @@ const express = require('express'); const { remote } = require('webdriverio'); +const multer = require('multer'); +const { execSync, exec } = require('child_process'); +const path = require('path'); +const fs = require('fs'); const APPIUM_HOST = process.env.APPIUM_HOST || '127.0.0.1'; const APPIUM_PORT = parseInt(process.env.APPIUM_PORT || '4723', 10); const PORT = parseInt(process.env.PORT || '3010', 10); const WEBHOOK_SECRET = process.env.WEBHOOK_SHARED_SECRET || 'dev-secret-change-me'; +const MEDIA_WAIT_MS = parseInt(process.env.MEDIA_WAIT_MS || '8000', 10); +const WHATSAPP_IMAGES_DIR = '/sdcard/Android/media/com.whatsapp/WhatsApp/Media/WhatsApp Images'; + +const upload = multer({ + dest: '/tmp/wa-uploads/', + limits: { fileSize: 20 * 1024 * 1024 }, + fileFilter: (_req, file, cb) => { + if (!file.mimetype.startsWith('image/')) { + return cb(new Error('Only image files are supported')); + } + cb(null, true); + }, +}); const app = express(); app.use(express.json({ limit: '10mb' })); @@ -55,6 +72,10 @@ function find(d, selector, timeout = 5000) { return d.$(selector).waitForExist({ timeout }).then(() => d.$(selector), () => null); } +function adb(args) { + return execSync(`adb ${args}`, { timeout: 30000, encoding: 'utf8' }).trim(); +} + async function ensureWhatsApp(d) { const pkg = await d.getCurrentPackage(); if (pkg !== 'com.whatsapp') { @@ -89,7 +110,6 @@ async function openChat(d, query) { await searchInput.setValue(query); await d.pause(2000); - // Find and click the matching chat row const chats = await d.$$(uia('resourceId("com.whatsapp:id/contact_row_container")')); for (const row of chats) { const nameEl = await row.$( @@ -102,7 +122,6 @@ async function openChat(d, query) { return; } } - // If no match by name, try clicking first result if (chats.length > 0) { await chats[0].click(); await d.pause(1500); @@ -111,16 +130,188 @@ async function openChat(d, query) { throw new Error(`No chat found for "${query}"`); } -// ── Debug ───────────────────────────────────────────────────────── +// ── ADB helpers ──────────────────────────────────────────────── +function adbPush(localPath, remotePath) { + return adb(`push "${localPath}" "${remotePath}"`); +} +function adbRm(remotePath) { + return adb(`shell rm -f "${remotePath}"`); +} -// ── Endpoints ───────────────────────────────────────────────────── +function adbShell(cmd) { + return adb(`shell ${cmd}`); +} + +function adbLatestFile(dir) { + const out = adb(`shell ls -t "${dir}/" 2>/dev/null | head -1`); + return out ? `${dir}/${out}` : null; +} + +function adbPull(remotePath, localPath) { + return adb(`pull "${remotePath}" "${localPath}"`); +} + +// ── Media send ───────────────────────────────────────────────── + +async function sendMediaViaShareIntent(d, chatName, localPath, mimetype, caption) { + const ext = path.extname(localPath) || '.jpg'; + const remoteName = `wa_${Date.now()}${ext}`; + const remotePath = `/sdcard/Pictures/${remoteName}`; + + adbPush(localPath, remotePath); + + try { + adbShell( + `am start -a android.intent.action.SEND ` + + `-t ${mimetype} ` + + `--eu android.intent.extra.STREAM file://${remotePath} ` + + `-p com.whatsapp` + ); + await d.pause(3000); + + const searchInput = await find( + d, + uia('resourceId("com.whatsapp:id/contactpicker_search")'), + 5000 + ); + if (!searchInput) { + throw new Error('Could not find contact picker search in share screen'); + } + + await searchInput.clearValue(); + await d.pause(300); + await searchInput.setValue(chatName); + await d.pause(2000); + + const contacts = await d.$$( + uia('resourceId("com.whatsapp:id/contactpicker_row_name")') + ); + let clicked = false; + for (const c of contacts) { + const name = await c.getText().catch(() => ''); + if (name.toLowerCase().includes(chatName.toLowerCase())) { + await c.click(); + clicked = true; + break; + } + } + if (!clicked && contacts.length > 0) { + await contacts[0].click(); + clicked = true; + } + if (!clicked) throw new Error(`Contact "${chatName}" not found in share picker`); + + await d.pause(3000); + + if (caption) { + const captionInput = await find( + d, + uia('resourceId("com.whatsapp:id/caption")'), + 3000 + ); + if (captionInput) { + await captionInput.clearValue(); + await d.pause(200); + await captionInput.setValue(caption); + await d.pause(500); + } else { + const entry = await find(d, uia('resourceId("com.whatsapp:id/entry")'), 2000); + if (entry) { + await entry.setValue(caption); + await d.pause(500); + } + } + } + + let sendBtn = await find(d, uia('resourceId("com.whatsapp:id/send")'), 5000); + if (!sendBtn) { + sendBtn = await find(d, uia('descriptionContains("Send")'), 3000); + } + if (!sendBtn) throw new Error('Could not find send button after selecting contact'); + + await sendBtn.click(); + await d.pause(2000); + + goHome(d).catch(() => {}); + } finally { + adbRm(remotePath); + } +} + +// ── Media receive ─────────────────────────────────────────────── + +function waitForNewImage(beforeTimestamp) { + const t0 = beforeTimestamp || Date.now(); + const maxWait = MEDIA_WAIT_MS; + const pollInterval = 1000; + let waited = 0; + + while (waited < maxWait) { + const latest = adbLatestFile(WHATSAPP_IMAGES_DIR); + if (latest) { + const tsMatch = adbShell(`stat -c '%Y' "${latest}" 2>/dev/null || echo 0`); + const ts = parseInt(tsMatch, 10) * 1000; + if (ts >= t0) return latest; + } + new (require('events').EventEmitter)().setMaxListeners(100); + const start = Date.now(); + while (Date.now() - start < pollInterval) {} + waited += pollInterval; + } + return null; +} + +function pullImageFile(remotePath) { + const filename = path.basename(remotePath); + const localPath = `/tmp/wa-downloads/${filename}`; + fs.mkdirSync('/tmp/wa-downloads', { recursive: true }); + adbPull(remotePath, localPath); + return localPath; +} + +let mediaTokenCounter = 0; +const mediaDownloads = new Map(); + +function fanoutMedia(event, localPath) { + const token = `media_${Date.now()}_${++mediaTokenCounter}`; + const filename = path.basename(localPath); + mediaDownloads.set(token, { path: localPath, filename, createdAt: Date.now() }); + + const downloadUrl = `http://127.0.0.1:${PORT}/download/${token}`; + const eventWithUrl = { ...event, mediaUrl: downloadUrl, mediaFilename: filename }; + + for (const sub of subscribers) { + fetch(sub.url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(eventWithUrl), + }).then((r) => { + console.log(` media fanout to ${sub.url}: ${r.status}`); + }).catch((e) => { + console.log(` media fanout to ${sub.url} failed: ${e.message}`); + }); + } +} + +// Cleanup old downloads every 5 min +setInterval(() => { + const cutoff = Date.now() - 10 * 60 * 1000; + for (const [token, entry] of mediaDownloads) { + if (entry.createdAt < cutoff) { + try { fs.unlinkSync(entry.path); } catch {} + mediaDownloads.delete(token); + } + } +}, 5 * 60 * 1000); + +// ── Endpoints (Appium) ───────────────────────────────────────── app.get('/health', async (_req, res) => { try { const d = await getDriver(); - const session = await d.getSession(); + await d.getSession(); res.json({ status: 'ok', sessionId: d.sessionId }); } catch (e) { res.status(503).json({ status: 'error', message: e.message }); @@ -194,9 +385,7 @@ app.post('/chat/fetchMessages/:session', async (req, res) => { for (const layout of layoutEls) { try { const textEl = await layout.$(uia('resourceId("com.whatsapp:id/message_text")')); - const dateEl = await layout.$( - uia('resourceId("com.whatsapp:id/date")') - ); + const dateEl = await layout.$(uia('resourceId("com.whatsapp:id/date")')); if (await textEl.isExisting()) { messages.push({ text: await textEl.getText(), @@ -246,7 +435,32 @@ app.post('/client/sendMessage/:session', async (req, res) => { } }); -// ── Webhook subscriber store (in-memory) ───────────────────── +// ── Media send endpoint ──────────────────────────────────────── + +app.post('/messages/send-media', upload.single('image'), async (req, res) => { + let localPath = null; + try { + const { chatName, caption } = req.body; + const file = req.file; + if (!chatName) return res.status(400).json({ error: 'chatName required' }); + if (!file) return res.status(400).json({ error: 'image file required' }); + + localPath = file.path; + const d = await getDriver(); + + await sendMediaViaShareIntent(d, chatName, localPath, file.mimetype, caption || ''); + + res.json({ status: 'sent', chatName, caption: caption || '' }); + } catch (e) { + res.status(500).json({ error: e.message }); + } finally { + if (localPath) { + try { fs.unlinkSync(localPath); } catch {} + } + } +}); + +// ── Webhook subscriber store ───────────────────────────────────── const subscribers = []; @@ -272,14 +486,43 @@ app.post('/webhooks/whatsapp/incoming', (req, res) => { return res.status(401).json({ error: 'invalid or missing webhook secret' }); } - const { sender, text, chatId, timestamp } = req.body; - if (!sender || !text || !chatId) { - return res.status(400).json({ error: 'sender, text, and chatId are required' }); + const { sender, text, chatId, timestamp, mediaType } = req.body; + if (!sender || !chatId) { + return res.status(400).json({ error: 'sender and chatId are required' }); } - const event = { sender, text, chatId, timestamp: timestamp || Date.now(), receivedAt: new Date().toISOString() }; - fanout(event); - res.json({ status: 'ok' }); + const event = { + sender, + text: text || '', + chatId, + timestamp: timestamp || Date.now(), + receivedAt: new Date().toISOString(), + mediaType: mediaType || 'text', + }; + + if (mediaType === 'image') { + res.json({ status: 'ok', mediaProcessing: 'pending' }); + + setImmediate(async () => { + try { + const imagePath = waitForNewImage(event.timestamp); + if (!imagePath) { + console.log(` media: no new image found for ${chatId} from ${sender}`); + fanout({ ...event, mediaStatus: 'not-found' }); + return; + } + const localPath = pullImageFile(imagePath); + console.log(` media: pulled ${imagePath} → ${localPath}`); + fanoutMedia({ ...event, mediaStatus: 'downloaded' }, localPath); + } catch (e) { + console.log(` media processing error: ${e.message}`); + fanout({ ...event, mediaStatus: 'error', error: e.message }); + } + }); + } else { + fanout({ ...event, mediaType: 'text' }); + res.json({ status: 'ok' }); + } }); app.post('/webhooks/subscribe', (req, res) => { @@ -296,9 +539,21 @@ app.get('/webhooks/subscribers', (_req, res) => { res.json({ subscribers: subscribers.map((s) => ({ url: s.url, createdAt: s.createdAt })) }); }); +// ── Media download endpoint ────────────────────────────────── + +app.get('/download/:token', (req, res) => { + const entry = mediaDownloads.get(req.params.token); + if (!entry) return res.status(404).json({ error: 'download expired or not found' }); + res.download(entry.path, entry.filename); +}); + // ── Server start ───────────────────────────────────────────── +fs.mkdirSync('/tmp/wa-uploads', { recursive: true }); +fs.mkdirSync('/tmp/wa-downloads', { recursive: true }); + app.listen(PORT, '0.0.0.0', () => { + const host = require('os').networkInterfaces(); console.log(`wa-appium-api listening on http://0.0.0.0:${PORT}`); console.log(` Appium server: ${APPIUM_HOST}:${APPIUM_PORT}`); console.log(` Webhook secret: ${WEBHOOK_SECRET === 'dev-secret-change-me' ? 'WARNING: using dev default' : 'configured'}`);