From 8ae0c0d4843993a25f6ec0e50e337ddc5e4e6219 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Mon, 29 May 2023 02:31:03 -0400 Subject: [PATCH] feat(reliability): WhatsApp Web version caching (#1847) * feat: use specific whatsapp web version * add constants to version updater * configurable caches * rename * typings * rm extraneous export * missing from the rename * rm version resolver * match readme version * use try-catch * only persist if cache miss --- .gitignore | 5 +++- index.d.ts | 22 ++++++++++++++ src/Client.js | 33 +++++++++++++++++++++ src/util/Constants.js | 4 +++ src/webCache/LocalWebCache.js | 43 ++++++++++++++++++++++++++++ src/webCache/RemoteWebCache.js | 40 ++++++++++++++++++++++++++ src/webCache/WebCache.js | 14 +++++++++ src/webCache/WebCacheFactory.js | 20 +++++++++++++ tools/version-checker/update-version | 25 ++++++++++------ 9 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 src/webCache/LocalWebCache.js create mode 100644 src/webCache/RemoteWebCache.js create mode 100644 src/webCache/WebCache.js create mode 100644 src/webCache/WebCacheFactory.js diff --git a/.gitignore b/.gitignore index 79f6467..12b53d4 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,7 @@ typings/ # Test sessions *session.json -.wwebjs_auth/ \ No newline at end of file +.wwebjs_auth/ + +# local version cache +.wwebjs_cache/ diff --git a/index.d.ts b/index.d.ts index 38d2ebb..4b686a6 100644 --- a/index.d.ts +++ b/index.d.ts @@ -365,6 +365,10 @@ declare namespace WAWebJS { puppeteer?: puppeteer.PuppeteerNodeLaunchOptions & puppeteer.ConnectOptions /** Determines how to save and restore sessions. Will use LegacySessionAuth if options.session is set. Otherwise, NoAuth will be used. */ authStrategy?: AuthStrategy, + /** The version of WhatsApp Web to use. Use options.webVersionCache to configure how the version is retrieved. */ + webVersion?: string, + /** Determines how to retrieve the WhatsApp Web version specified in options.webVersion. */ + webVersionCache?: WebCacheOptions, /** How many times should the qrcode be refreshed before giving up * @default 0 (disabled) */ qrMaxRetries?: number, @@ -392,6 +396,24 @@ declare namespace WAWebJS { proxyAuthentication?: {username: string, password: string} | undefined } + export interface LocalWebCacheOptions { + type: 'local', + path?: string, + strict?: boolean + } + + export interface RemoteWebCacheOptions { + type: 'remote', + remotePath: string, + strict?: boolean + } + + export interface NoWebCacheOptions { + type: 'none' + } + + export type WebCacheOptions = NoWebCacheOptions | LocalWebCacheOptions | RemoteWebCacheOptions; + /** * Base class which all authentication strategies extend */ diff --git a/src/Client.js b/src/Client.js index fef8e4c..d59bac6 100644 --- a/src/Client.js +++ b/src/Client.js @@ -10,6 +10,7 @@ const { WhatsWebURL, DefaultOptions, Events, WAState } = require('./util/Constan const { ExposeStore, LoadUtils } = require('./util/Injected'); const ChatFactory = require('./factories/ChatFactory'); const ContactFactory = require('./factories/ContactFactory'); +const WebCacheFactory = require('./webCache/WebCacheFactory'); const { ClientInfo, Message, MessageMedia, Contact, Location, GroupNotification, Label, Call, Buttons, List, Reaction, Chat } = require('./structures'); const LegacySessionAuth = require('./authStrategies/LegacySessionAuth'); const NoAuth = require('./authStrategies/NoAuth'); @@ -19,6 +20,8 @@ const NoAuth = require('./authStrategies/NoAuth'); * @extends {EventEmitter} * @param {object} options - Client options * @param {AuthStrategy} options.authStrategy - Determines how to save and restore sessions. Will use LegacySessionAuth if options.session is set. Otherwise, NoAuth will be used. + * @param {string} options.webVersion - The version of WhatsApp Web to use. Use options.webVersionCache to configure how the version is retrieved. + * @param {object} options.webVersionCache - Determines how to retrieve the WhatsApp Web version. Defaults to a local cache (LocalWebCache) that falls back to latest if the requested version is not found. * @param {number} options.authTimeoutMs - Timeout for authentication selector in puppeteer * @param {object} options.puppeteer - Puppeteer launch options. View docs here: https://github.com/puppeteer/puppeteer/ * @param {number} options.qrMaxRetries - How many times should the qrcode be refreshed before giving up @@ -115,6 +118,7 @@ class Client extends EventEmitter { this.pupPage = page; await this.authStrategy.afterBrowserInitialized(); + await this.initWebVersionCache(); await page.goto(WhatsWebURL, { waitUntil: 'load', @@ -638,6 +642,35 @@ class Client extends EventEmitter { }); } + async initWebVersionCache() { + const { type: webCacheType, ...webCacheOptions } = this.options.webVersionCache; + const webCache = WebCacheFactory.createWebCache(webCacheType, webCacheOptions); + + const requestedVersion = this.options.webVersion; + const versionContent = await webCache.resolve(requestedVersion); + + if(versionContent) { + await this.pupPage.setRequestInterception(true); + this.pupPage.on('request', async (req) => { + if(req.url() === WhatsWebURL) { + req.respond({ + status: 200, + contentType: 'text/html', + body: versionContent + }); + } else { + req.continue(); + } + }); + } else { + this.pupPage.on('response', async (res) => { + if(res.ok() && res.url() === WhatsWebURL) { + await webCache.persist(await res.text()); + } + }); + } + } + /** * Closes the client */ diff --git a/src/util/Constants.js b/src/util/Constants.js index aa1005d..96de713 100644 --- a/src/util/Constants.js +++ b/src/util/Constants.js @@ -7,6 +7,10 @@ exports.DefaultOptions = { headless: true, defaultViewport: null }, + webVersion: '2.2306.7', + webVersionCache: { + type: 'local', + }, authTimeoutMs: 0, qrMaxRetries: 0, takeoverOnConflict: false, diff --git a/src/webCache/LocalWebCache.js b/src/webCache/LocalWebCache.js new file mode 100644 index 0000000..a377f55 --- /dev/null +++ b/src/webCache/LocalWebCache.js @@ -0,0 +1,43 @@ +const path = require('path'); +const fs = require('fs'); + +const { WebCache, VersionResolveError } = require('./WebCache'); + +/** + * LocalWebCache - Fetches a WhatsApp Web version from a local file store + * @param {object} options - options + * @param {string} options.path - Path to the directory where cached versions are saved, default is: "./.wwebjs_cache/" + * @param {boolean} options.strict - If true, will throw an error if the requested version can't be fetched. If false, will resolve to the latest version. + */ +class LocalWebCache extends WebCache { + constructor(options = {}) { + super(); + + this.path = options.path || './.wwebjs_cache/'; + this.strict = options.strict || false; + } + + async resolve(version) { + const filePath = path.join(this.path, `${version}.html`); + + try { + return fs.readFileSync(filePath, 'utf-8'); + } + catch (err) { + if (this.strict) throw new VersionResolveError(`Couldn't load version ${version} from the cache`); + return null; + } + } + + async persist(indexHtml) { + // extract version from index (e.g. manifest-2.2206.9.json -> 2.2206.9) + const version = indexHtml.match(/manifest-([\d\\.]+)\.json/)[1]; + if(!version) return; + + const filePath = path.join(this.path, `${version}.html`); + fs.mkdirSync(this.path, { recursive: true }); + fs.writeFileSync(filePath, indexHtml); + } +} + +module.exports = LocalWebCache; \ No newline at end of file diff --git a/src/webCache/RemoteWebCache.js b/src/webCache/RemoteWebCache.js new file mode 100644 index 0000000..04da0f1 --- /dev/null +++ b/src/webCache/RemoteWebCache.js @@ -0,0 +1,40 @@ +const fetch = require('node-fetch'); +const { WebCache, VersionResolveError } = require('./WebCache'); + +/** + * RemoteWebCache - Fetches a WhatsApp Web version index from a remote server + * @param {object} options - options + * @param {string} options.remotePath - Endpoint that should be used to fetch the version index. Use {version} as a placeholder for the version number. + * @param {boolean} options.strict - If true, will throw an error if the requested version can't be fetched. If false, will resolve to the latest version. Defaults to false. + */ +class RemoteWebCache extends WebCache { + constructor(options = {}) { + super(); + + if (!options.remotePath) throw new Error('webVersionCache.remotePath is required when using the remote cache'); + this.remotePath = options.remotePath; + this.strict = options.strict || false; + } + + async resolve(version) { + const remotePath = this.remotePath.replace('{version}', version); + + try { + const cachedRes = await fetch(remotePath); + if (cachedRes.ok) { + return cachedRes.text(); + } + } catch (err) { + console.error(`Error fetching version ${version} from remote`, err); + } + + if (this.strict) throw new VersionResolveError(`Couldn't load version ${version} from the archive`); + return null; + } + + async persist() { + // Nothing to do here + } +} + +module.exports = RemoteWebCache; \ No newline at end of file diff --git a/src/webCache/WebCache.js b/src/webCache/WebCache.js new file mode 100644 index 0000000..4460218 --- /dev/null +++ b/src/webCache/WebCache.js @@ -0,0 +1,14 @@ +/** + * Default implementation of a web version cache that does nothing. + */ +class WebCache { + async resolve() { return null; } + async persist() { } +} + +class VersionResolveError extends Error { } + +module.exports = { + WebCache, + VersionResolveError +}; \ No newline at end of file diff --git a/src/webCache/WebCacheFactory.js b/src/webCache/WebCacheFactory.js new file mode 100644 index 0000000..2c4fda7 --- /dev/null +++ b/src/webCache/WebCacheFactory.js @@ -0,0 +1,20 @@ +const RemoteWebCache = require('./RemoteWebCache'); +const LocalWebCache = require('./LocalWebCache'); +const { WebCache } = require('./WebCache'); + +const createWebCache = (type, options) => { + switch (type) { + case 'remote': + return new RemoteWebCache(options); + case 'local': + return new LocalWebCache(options); + case 'none': + return new WebCache(); + default: + throw new Error(`Invalid WebCache type ${type}`); + } +}; + +module.exports = { + createWebCache, +}; \ No newline at end of file diff --git a/tools/version-checker/update-version b/tools/version-checker/update-version index 3afbb27..dc68d23 100755 --- a/tools/version-checker/update-version +++ b/tools/version-checker/update-version @@ -18,15 +18,24 @@ const getCurrentVersion = () => { } }; -const updateVersion = async (oldVersion, newVersion) => { - const readmePath = '../../README.md'; - - const readme = fs.readFileSync(readmePath); - const newReadme = readme.toString().replaceAll(oldVersion, newVersion); +const updateInFile = (filePath, oldVersion, newVersion) => { + const originalFile = fs.readFileSync(filePath); + const newFile = originalFile.toString().replaceAll(oldVersion, newVersion); - fs.writeFileSync(readmePath, newReadme); - fs.writeFileSync('./.version', newVersion); - + fs.writeFileSync(filePath, newFile); +}; + +const updateVersion = async (oldVersion, newVersion) => { + const filesToUpdate = [ + '../../src/util/Constants.js', + '../../README.md', + ]; + + for (const file of filesToUpdate) { + updateInFile(file, oldVersion, newVersion); + } + + fs.writeFileSync('./.version', newVersion); }; (async () => {