diff --git a/package-lock.json b/package-lock.json index f0068615..5271538b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5087,9 +5087,9 @@ } }, "binary-extensions": { - "version": "2.0.0", - "resolved": "https://registry.npm.taobao.org/binary-extensions/download/binary-extensions-2.0.0.tgz", - "integrity": "sha1-I8DfFPaogHf1+YbA0WfsA8PVU3w=" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==" }, "bindings": { "version": "1.5.0", @@ -5879,9 +5879,9 @@ } }, "chokidar": { - "version": "3.4.0", - "resolved": "https://registry.npm.taobao.org/chokidar/download/chokidar-3.4.0.tgz", - "integrity": "sha1-swYRQjzjdjV8dlubj5BLn7o8C+g=", + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.1.tgz", + "integrity": "sha512-TQTJyr2stihpC4Sya9hs2Xh+O2wf+igjL36Y75xx2WdHuiICcn/XJza46Jwt0eT5hVpQOzo3FpY3cj3RVYLX0g==", "requires": { "anymatch": "~3.1.1", "braces": "~3.0.2", @@ -5895,8 +5895,8 @@ "dependencies": { "anymatch": { "version": "3.1.1", - "resolved": "https://registry.npm.taobao.org/anymatch/download/anymatch-3.1.1.tgz", - "integrity": "sha1-xV7PAhheJGklk5kxDBc84xIzsUI=", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -5904,34 +5904,34 @@ }, "braces": { "version": "3.0.2", - "resolved": "https://registry.npm.taobao.org/braces/download/braces-3.0.2.tgz", - "integrity": "sha1-NFThpGLujVmeI23zNs2epPiv4Qc=", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", "requires": { "fill-range": "^7.0.1" } }, "fill-range": { "version": "7.0.1", - "resolved": "https://registry.npm.taobao.org/fill-range/download/fill-range-7.0.1.tgz", - "integrity": "sha1-GRmmp8df44ssfHflGYU12prN2kA=", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", "requires": { "to-regex-range": "^5.0.1" } }, "is-number": { "version": "7.0.0", - "resolved": "https://registry.npm.taobao.org/is-number/download/is-number-7.0.0.tgz", - "integrity": "sha1-dTU0W4lnNNX4DE0GxQlVUnoU8Ss=" + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "normalize-path": { "version": "3.0.0", - "resolved": "https://registry.npm.taobao.org/normalize-path/download/normalize-path-3.0.0.tgz", - "integrity": "sha1-Dc1p/yOhybEf0JeDFmRKA4ghamU=" + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, "to-regex-range": { "version": "5.0.1", - "resolved": "https://registry.npm.taobao.org/to-regex-range/download/to-regex-range-5.0.1.tgz", - "integrity": "sha1-FkjESq58jZiKMmAY7XL1tN0DkuQ=", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", "requires": { "is-number": "^7.0.0" } @@ -11365,8 +11365,8 @@ }, "is-binary-path": { "version": "2.1.0", - "resolved": "https://registry.npm.taobao.org/is-binary-path/download/is-binary-path-2.1.0.tgz", - "integrity": "sha1-6h9/O4DwZCNug0cPhsCcJU+0Wwk=", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", "requires": { "binary-extensions": "^2.0.0" } @@ -14661,8 +14661,8 @@ }, "picomatch": { "version": "2.2.2", - "resolved": "https://registry.npm.taobao.org/picomatch/download/picomatch-2.2.2.tgz", - "integrity": "sha1-IfMz6ba46v8CRo9RRupAbTRfTa0=" + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==" }, "pify": { "version": "4.0.1", @@ -16779,8 +16779,8 @@ }, "readdirp": { "version": "3.4.0", - "resolved": "https://registry.npm.taobao.org/readdirp/download/readdirp-3.4.0.tgz?cache=0&sync_timestamp=1584985910691&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Freaddirp%2Fdownload%2Freaddirp-3.4.0.tgz", - "integrity": "sha1-n9zN+ekVWAVEkiGsZF6DA6tbmto=", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", + "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", "requires": { "picomatch": "^2.2.1" } diff --git a/package.json b/package.json index 51f83a4e..5f5c06ea 100755 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "ace-builds": "1.4.12", "blueimp-md5": "2.16.0", "cheerio": "1.0.0-rc.3", + "chokidar": "^3.4.1", "classnames": "2.2.6", "darkreader": "4.9.15", "date-fns": "2.14.0", diff --git a/public/libs/wiki/watch-wiki-worker.js b/public/libs/wiki/watch-wiki-worker.js new file mode 100644 index 00000000..9ed27088 --- /dev/null +++ b/public/libs/wiki/watch-wiki-worker.js @@ -0,0 +1,75 @@ +const { workerData, parentPort, isMainThread } = require('worker_threads'); +const fs = require('fs'); +const path = require('path'); +const chokidar = require('chokidar'); +const { trim, compact } = require('lodash'); + +const { commitAndSync } = require('../git'); + +const frequentlyChangedFileThatShouldBeIgnoredFromWatch = ['output', /\$__StoryList/]; +const topLevelFoldersToIgnored = ['node_modules', '.git']; + +/** https://davidwalsh.name/javascript-debounce-function */ +function debounce(func, wait, immediate) { + let timeout; + return function debounced() { + const context = this; + // eslint-disable-next-line no-underscore-dangle, prefer-rest-params + const arguments_ = arguments; + const later = function later() { + timeout = undefined; + if (!immediate) func.apply(context, arguments_); + }; + const callNow = immediate && !timeout; + clearTimeout(timeout); + timeout = setTimeout(later, wait); + if (callNow) func.apply(context, arguments_); + }; +} + +let watcher; +function watchFolder(wikiRepoPath, wikiFolderPath, githubRepoUrl, userInfo, syncDebounceInterval) { + const debounceCommitAndSync = debounce(commitAndSync, syncDebounceInterval); + // simple lock to prevent running two instance of commit task + let lock = false; + // load ignore config from .gitignore located in the wiki repo folder + const gitIgnoreFilePath = path.join(wikiRepoPath, '.gitignore'); + let gitignoreFile = ''; + try { + gitignoreFile = fs.readFileSync(gitIgnoreFilePath, 'utf-8') || ''; + } catch (error) { + parentPort.postMessage(`Error: fail to load .gitignore from ${gitIgnoreFilePath} \n ${error} ${error.stack}`); + } + const filesToIgnoreFromGitIgnore = compact(gitignoreFile.split('\n').filter(line => !trim(line).startsWith('#'))); + watcher = chokidar.watch(wikiFolderPath, { + ignored: [ + ...filesToIgnoreFromGitIgnore, + ...topLevelFoldersToIgnored, + ...frequentlyChangedFileThatShouldBeIgnoredFromWatch, + ], + cwd: wikiFolderPath, + }); + watcher.on( + 'all', + debounce(async (_, fileName) => { + if (lock) { + parentPort.postMessage(`${fileName} changed, but lock is on, so skip`); + return; + } + parentPort.postMessage(`${fileName} changed`); + lock = true; + await debounceCommitAndSync(wikiRepoPath, githubRepoUrl, userInfo); + lock = false; + }, 1000), + ); + parentPort.postMessage(`wiki Github syncer is watching ${wikiFolderPath} now`); +} + +function watchWiki() { + const { wikiRepoPath, githubRepoUrl, userInfo, wikiFolderPath, syncDebounceInterval } = workerData; + watchFolder(wikiRepoPath, wikiFolderPath, githubRepoUrl, userInfo, syncDebounceInterval); +} + +if (!isMainThread) { + watchWiki(); +} diff --git a/public/libs/wiki/watch-wiki.js b/public/libs/wiki/watch-wiki.js index 28173e9c..69c80bcd 100644 --- a/public/libs/wiki/watch-wiki.js +++ b/public/libs/wiki/watch-wiki.js @@ -1,47 +1,11 @@ const fs = require('fs'); -const { commitAndSync } = require('../git'); +const { startWikiWatcher } = require('./wiki-worker-mamager'); const { getPreference } = require('../preferences'); -const frequentlyChangedFileThatShouldBeIgnoredFromWatch = new Set(['output', '$__StoryList.tid']); -const topLevelFoldersToIgnored = ['node_modules', '.git']; - -/** https://davidwalsh.name/javascript-debounce-function */ -function debounce(func, wait, immediate) { - let timeout; - return function debounced() { - const context = this; - // eslint-disable-next-line no-underscore-dangle, prefer-rest-params - const arguments_ = arguments; - const later = function later() { - timeout = undefined; - if (!immediate) func.apply(context, arguments_); - }; - const callNow = immediate && !timeout; - clearTimeout(timeout); - timeout = setTimeout(later, wait); - if (callNow) func.apply(context, arguments_); - }; -} - -const debounceCommitAndSync = debounce(commitAndSync, getPreference('syncDebounceInterval')); - -function watchFolder(wikiRepoPath, wikiFolderPath, githubRepoUrl, userInfo) { - fs.watch( - wikiFolderPath, - { recursive: true }, - debounce((_, fileName) => { - if (topLevelFoldersToIgnored.some(name => fileName.startsWith(name))) return; - if (frequentlyChangedFileThatShouldBeIgnoredFromWatch.has(fileName)) return; - console.log(`${fileName} change`); - debounceCommitAndSync(wikiRepoPath, githubRepoUrl, userInfo); - }, 1000), - ); - console.log(`wiki watch ${wikiFolderPath} now`); -} - module.exports = function watchWiki(wikiRepoPath, githubRepoUrl, userInfo, wikiFolderPath = wikiRepoPath) { if (fs.existsSync(wikiRepoPath)) { - watchFolder(wikiRepoPath, wikiFolderPath, githubRepoUrl, userInfo); + const syncDebounceInterval = getPreference('syncDebounceInterval') + startWikiWatcher(wikiRepoPath, githubRepoUrl, userInfo, wikiFolderPath, syncDebounceInterval); } }; diff --git a/public/libs/wiki/wiki-worker-mamager.js b/public/libs/wiki/wiki-worker-mamager.js index 1c79dd9a..361bb571 100644 --- a/public/libs/wiki/wiki-worker-mamager.js +++ b/public/libs/wiki/wiki-worker-mamager.js @@ -4,26 +4,46 @@ const isDev = require('electron-is-dev'); const path = require('path'); -const WIKI_WORKER_PATH = isDev -? path.resolve(__dirname, './wiki-worker.js') -: path.resolve(process.resourcesPath, '..', 'wiki-worker.js'); - +// key is same to workspace name, so we can get this worker by workspace name // { [name: string]: Worker } -const workers = {}; +const wikiWorkers = {}; +const wikiWatcherWorkers = {}; module.exports.startWiki = function startWiki(homePath, tiddlyWikiPort, userName) { + const WIKI_WORKER_PATH = isDev + ? path.resolve(__dirname, './wiki-worker.js') + : path.resolve(process.resourcesPath, '..', 'wiki-worker.js'); const workerData = { homePath, userName, tiddlyWikiPort }; const worker = new Worker(WIKI_WORKER_PATH, { workerData }); - workers[homePath] = worker; - worker.on('message', message => console.log(`[${homePath}] ${message}`)); - worker.on('error', error => console.error(`[${homePath}] ${error}`)); + wikiWorkers[homePath] = worker; + worker.on('message', message => console.log(`[NodeJSWiki ${homePath}] ${message}`)); + worker.on('error', error => console.error(`[NodeJSWiki ${homePath}] ${error}`)); worker.on('exit', code => { - if (code !== 0) console.error(`[${homePath}] Worker stopped with exit code ${code}`); + if (code !== 0) console.error(`[NodeJSWiki ${homePath}] Worker stopped with exit code ${code}`); }); }; - module.exports.stopWiki = function stopWiki(homePath) { - const worker = workers[homePath]; + const worker = wikiWorkers[homePath]; + if (!worker) return; // no running worker, maybe tiddlywiki server in this workspace failed to start + worker.terminate(); +}; + +module.exports.startWikiWatcher = function startWikiWatcher(wikiRepoPath, githubRepoUrl, userInfo, wikiFolderPath, syncDebounceInterval) { + const WIKI_WATCHER_WORKER_PATH = isDev + ? path.resolve(__dirname, './watch-wiki-worker.js') + : path.resolve(process.resourcesPath, '..', 'watch-wiki-worker.js'); + const workerData = { wikiRepoPath, githubRepoUrl, userInfo, wikiFolderPath, syncDebounceInterval }; + const worker = new Worker(WIKI_WATCHER_WORKER_PATH, { workerData }); + wikiWatcherWorkers[wikiRepoPath] = worker; + worker.on('message', message => console.log(`[WikiWatcher ${wikiRepoPath}] ${message}`)); + worker.on('error', error => console.error(`[WikiWatcher ${wikiRepoPath}] ${error}`)); + worker.on('exit', code => { + if (code !== 0) console.error(`[WikiWatcher ${wikiRepoPath}] Worker stopped with exit code ${code}`); + }); +}; + +module.exports.stopWikiWatcher = function stopWikiWatcher(wikiRepoPath) { + const worker = wikiWatcherWorkers[wikiRepoPath]; if (!worker) return; // no running worker, maybe tiddlywiki server in this workspace failed to start worker.terminate(); }; diff --git a/public/libs/workspaces.js b/public/libs/workspaces.js index 8e4f48cd..6c9f9773 100644 --- a/public/libs/workspaces.js +++ b/public/libs/workspaces.js @@ -9,7 +9,7 @@ const download = require('download'); const tmp = require('tmp'); const sendToAllWindows = require('./send-to-all-windows'); -const { stopWiki } = require('./wiki/wiki-worker-mamager'); +const { stopWiki, stopWikiWatcher } = require('./wiki/wiki-worker-mamager'); const v = '14'; @@ -170,6 +170,7 @@ const removeWorkspacePicture = (id) => { const removeWorkspace = (id) => { const { name } = workspaces[id]; stopWiki(name); + stopWikiWatcher(name); delete workspaces[id]; sendToAllWindows('set-workspace', id, null); settings.unsetSync(`workspaces.${v}.${id}`);