diff --git a/.gitignore b/.gitignore index 5fcc3844..0274326e 100644 --- a/.gitignore +++ b/.gitignore @@ -68,3 +68,4 @@ https-keys-dev/ .cache/ deb2appimage_cache/ deb2appimage.json +cache-database-dev/ \ No newline at end of file diff --git a/package.json b/package.json index 978b212d..ed26fbf9 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,7 @@ "make:win-arm": "npm run init:git-submodule && cross-env NODE_ENV=production electron-forge make --platform=win32 --arch=arm64", "make:linux-x64": "npm run init:git-submodule && cross-env NODE_ENV=production electron-forge make --platform=linux --arch=x64", "make:linux-arm": "npm run init:git-submodule && cross-env NODE_ENV=production electron-forge make --platform=linux --arch=arm64", - "clean": "rimraf ./out ./settings-dev ./logs ./.webpack && cross-env NODE_ENV=development npx ts-node scripts/developmentMkdir.ts && npm run init:git-submodule", + "clean": "rimraf ./out ./settings-dev ./cache-database-dev ./logs ./.webpack && cross-env NODE_ENV=development npx ts-node scripts/developmentMkdir.ts && npm run init:git-submodule", "init:git-submodule": "git submodule update --recursive && git submodule update --remote", "lint": "eslint ./src --ext js,ts,tsx,json", "lint:fix": "eslint ./src --ext js,ts,tsx,json --fix", @@ -30,6 +30,7 @@ "@tiddlygit/tiddlywiki": "^5.3.0-prerelease-2023-05-22", "app-path": "4.0.0", "best-effort-json-parser": "1.0.1", + "better-sqlite3": "^8.4.0", "bluebird": "3.7.2", "default-gateway": "6.0.3", "dugite": "^2.5.0", @@ -61,6 +62,7 @@ "rxjs": "7.8.1", "semver": "7.5.1", "source-map-support": "0.5.21", + "sqlite-vss": "0.1.1-alpha.13", "strip-ansi": "^7.0.1", "threads": "1.7.0", "type-fest": "3.10.0", @@ -71,6 +73,11 @@ "winston-transport": "4.5.0", "zx": "7.2.2" }, + "optionalDependencies": { + "sqlite-vss-darwin-x64": "0.1.1-alpha.13", + "sqlite-vss-darwin-arm64": "0.1.1-alpha.13", + "sqlite-vss-linux-x64": "0.1.1-alpha.13" + }, "devDependencies": { "@cucumber/cucumber": "9.1.2", "@dnd-kit/core": "4.0.0", @@ -93,6 +100,7 @@ "@material-ui/lab": "5.0.0-alpha.27", "@material-ui/styled-engine-sc": "5.0.0-alpha.26", "@reforged/maker-appimage": "^3.2.0", + "@types/better-sqlite3": "^7.6.4", "@types/bluebird": "3.5.38", "@types/chai": "4.3.5", "@types/circular-dependency-plugin": "5.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf001155..983d1192 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,6 +10,9 @@ dependencies: best-effort-json-parser: specifier: 1.0.1 version: 1.0.1 + better-sqlite3: + specifier: ^8.4.0 + version: 8.4.0 bluebird: specifier: 3.7.2 version: 3.7.2 @@ -103,6 +106,9 @@ dependencies: source-map-support: specifier: 0.5.21 version: 0.5.21 + sqlite-vss: + specifier: 0.1.1-alpha.13 + version: 0.1.1-alpha.13 strip-ansi: specifier: ^7.0.1 version: 7.0.1 @@ -131,6 +137,17 @@ dependencies: specifier: 7.2.2 version: 7.2.2 +optionalDependencies: + sqlite-vss-darwin-arm64: + specifier: 0.1.1-alpha.13 + version: 0.1.1-alpha.13 + sqlite-vss-darwin-x64: + specifier: 0.1.1-alpha.13 + version: 0.1.1-alpha.13 + sqlite-vss-linux-x64: + specifier: 0.1.1-alpha.13 + version: 0.1.1-alpha.13 + devDependencies: '@cucumber/cucumber': specifier: 9.1.2 @@ -195,6 +212,9 @@ devDependencies: '@reforged/maker-appimage': specifier: ^3.2.0 version: 3.2.0(bluebird@3.7.2) + '@types/better-sqlite3': + specifier: ^7.6.4 + version: 7.6.4 '@types/bluebird': specifier: 3.5.38 version: 3.5.38 @@ -2479,6 +2499,12 @@ packages: resolution: {integrity: sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==} dev: true + /@types/better-sqlite3@7.6.4: + resolution: {integrity: sha512-dzrRZCYPXIXfSR1/surNbJ/grU3scTaygS0OMzjlGf71i9sc2fGyHPXXiXmEvNIoE0cGwsanEFMVJxPXmco9Eg==} + dependencies: + '@types/node': 20.2.1 + dev: true + /@types/bluebird@3.5.38: resolution: {integrity: sha512-yR/Kxc0dd4FfwtEoLZMoqJbM/VE/W7hXn/MIjb+axcwag0iFmSPK7OBUZq1YWLynJUoWQkfUrI7T0HDqGApNSg==} dev: true @@ -3652,6 +3678,14 @@ packages: resolution: {integrity: sha512-Fbne5nLqmDQXUmHf6cxeFAAIcsiFaSbCBoK0Rt+wgWv/7SacNhViNeXnuf0H0wXupwW5Io8FfB1FHPUfZujwwg==} dev: false + /better-sqlite3@8.4.0: + resolution: {integrity: sha512-NmsNW1CQvqMszu/CFAJ3pLct6NEFlNfuGM6vw72KHkjOD1UDnL96XNN1BMQc1hiHo8vE2GbOWQYIpZ+YM5wrZw==} + requiresBuild: true + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.1 + dev: false + /big-integer@1.6.51: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} @@ -3666,6 +3700,12 @@ packages: engines: {node: '>=8'} dev: true + /bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + dependencies: + file-uri-to-path: 1.0.0 + dev: false + /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} dependencies: @@ -4706,7 +4746,6 @@ packages: /detect-libc@2.0.1: resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} engines: {node: '>=8'} - dev: true /detect-node@2.1.0: resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} @@ -6048,6 +6087,10 @@ packages: token-types: 4.2.1 dev: false + /file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + dev: false + /filename-reserved-regex@2.0.0: resolution: {integrity: sha1-q/c9+rc10EVECr/qLZHzieu/oik=} engines: {node: '>=4'} @@ -8274,7 +8317,6 @@ packages: engines: {node: '>=10'} dependencies: semver: 7.5.1 - dev: true /node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} @@ -9040,6 +9082,25 @@ packages: which-pm-runs: 1.1.0 dev: false + /prebuild-install@7.1.1: + resolution: {integrity: sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + detect-libc: 2.0.1 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 1.0.2 + node-abi: 3.40.0 + pump: 3.0.0 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.1 + tunnel-agent: 0.6.0 + dev: false + /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} @@ -10086,6 +10147,38 @@ packages: resolution: {integrity: sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==} optional: true + /sqlite-vss-darwin-arm64@0.1.1-alpha.13: + resolution: {integrity: sha512-jakStdR47vBW2zuZuRprrMFe+mz6coAufaTArNid3SdvyqwfzWT5QnTR0iM3LDVDChBrpVCreEqxwc0669sUwg==} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /sqlite-vss-darwin-x64@0.1.1-alpha.13: + resolution: {integrity: sha512-DI0Yoiw7NwiHuQSmXy7T1iRaE97MzAhtTFdnOwQevJahSr6ojziwsdZEuvfwjPP/hpL1xcVazsJBwXdSnAgypg==} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /sqlite-vss-linux-x64@0.1.1-alpha.13: + resolution: {integrity: sha512-+dDOvhiFQoXrecSqrh3dBTDO1ks43m0gS08agyBLEHByN/5mM7MzXltKBWf5oNsS9oZ6rsr2UzxQKZszs9hVmA==} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /sqlite-vss@0.1.1-alpha.13: + resolution: {integrity: sha512-e932Qv2eaycaWI/k8cGvsKB++p6df9UHvNdw93vJM6AX/Uy/H2t764QE4a/CuJPi34W6VbtKXTXmGbQQP8dduw==} + optionalDependencies: + sqlite-vss-darwin-arm64: 0.1.1-alpha.13 + sqlite-vss-darwin-x64: 0.1.1-alpha.13 + sqlite-vss-linux-x64: 0.1.1-alpha.13 + dev: false + /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} diff --git a/scripts/afterPack.js b/scripts/afterPack.js index 94f78979..16e66384 100644 --- a/scripts/afterPack.js +++ b/scripts/afterPack.js @@ -43,6 +43,13 @@ exports.default = async (buildPath, electronVersion, platform, arch, callback) = // we only need its `main` binary await fs.mkdirp(path.join(cwd, 'node_modules', 'app-path')); await fs.copy(path.join(projectRoot, 'node_modules', 'app-path', 'main'), path.join(cwd, 'node_modules', 'app-path', 'main'), { dereference: true }); + await fs.copy(path.resolve(projectRoot, 'node_modules/better-sqlite3/build/Release/better_sqlite3.node'), path.resolve(cwd, 'node_modules/better-sqlite3/build/Release/better_sqlite3.node'), { dereference: true }); + const sqliteVssPackages = ['sqlite-vss', 'sqlite-vss-linux-x64', 'sqlite-vss-darwin-x64', 'sqlite-vss-darwin-arm64'] + for (const sqliteVssPackage of sqliteVssPackages) { + try { + await fs.copy(path.resolve(projectRoot, `node_modules/${sqliteVssPackage}`), path.resolve(cwd, `node_modules/${sqliteVssPackage}`), { dereference: true }); + } catch {} + } // await exec(`npm i --legacy-building`, { cwd: path.join(cwd, 'node_modules', 'app-path') }); // await exec(`npm i --legacy-building`, { cwd: path.join(cwd, 'node_modules', 'app-path', 'node_modules', 'cross-spawn') }); // await exec(`npm i --legacy-building`, { cwd: path.join(cwd, 'node_modules', 'app-path', 'node_modules', 'get-stream') }); diff --git a/src/constants/appPaths.ts b/src/constants/appPaths.ts index 0f47b14a..12821768 100644 --- a/src/constants/appPaths.ts +++ b/src/constants/appPaths.ts @@ -2,18 +2,20 @@ import { app } from 'electron'; import path from 'path'; import { __TEST__ as v8CompileCacheLibrary } from 'v8-compile-cache-lib'; import { isDevelopmentOrTest } from './environment'; -import { developmentHttpsCertKeyFolderName, developmentSettingFolderName } from './fileNames'; +import { cacheDatabaseFolderName, httpsCertKeyFolderName, settingFolderName } from './fileNames'; import { sourcePath } from './paths'; export const USER_DATA_FOLDER = app.getPath('userData'); export const SETTINGS_FOLDER = isDevelopmentOrTest - ? path.resolve(sourcePath, '..', developmentSettingFolderName) - // eslint-disable-next-line @typescript-eslint/no-var-requires - : path.resolve(USER_DATA_FOLDER, 'settings'); + /** Used to store settings during dev and testing */ + ? path.resolve(sourcePath, '..', `${settingFolderName}-dev`) + : path.resolve(USER_DATA_FOLDER, settingFolderName); export const HTTPS_CERT_KEY_FOLDER = isDevelopmentOrTest - ? path.resolve(sourcePath, '..', developmentHttpsCertKeyFolderName) - // eslint-disable-next-line @typescript-eslint/no-var-requires - : path.resolve(USER_DATA_FOLDER, 'https-keys'); + ? path.resolve(sourcePath, '..', `${httpsCertKeyFolderName}-dev`) + : path.resolve(USER_DATA_FOLDER, httpsCertKeyFolderName); +export const CACHE_DATABASE_FOLDER = isDevelopmentOrTest + ? path.resolve(sourcePath, '..', `${cacheDatabaseFolderName}-dev`) + : path.resolve(USER_DATA_FOLDER, cacheDatabaseFolderName); export const LOCAL_GIT_DIRECTORY = path.resolve(isDevelopmentOrTest ? path.join(sourcePath, '..') : process.resourcesPath, 'node_modules', 'dugite', 'git'); export const LOG_FOLDER = isDevelopmentOrTest ? path.resolve(sourcePath, '..', 'logs') : path.resolve(USER_DATA_FOLDER, 'logs'); export const V8_CACHE_FOLDER = v8CompileCacheLibrary.getCacheDir(); diff --git a/src/constants/channels.ts b/src/constants/channels.ts index 1fe23368..4104c7ef 100644 --- a/src/constants/channels.ts +++ b/src/constants/channels.ts @@ -14,6 +14,12 @@ export enum AuthenticationChannel { export enum ContextChannel { name = 'ContextChannel', } +export enum DatabaseChannel { + getTiddlers = 'get-tiddlers', + insertTiddlers = 'insert-tiddlers', + name = 'DatabaseChannel', + searchTiddlers = 'search-tiddlers', +} export enum GitChannel { name = 'GitChannel', } diff --git a/src/constants/fileNames.ts b/src/constants/fileNames.ts index 05b35ae5..468d74c8 100644 --- a/src/constants/fileNames.ts +++ b/src/constants/fileNames.ts @@ -1,6 +1,6 @@ -/** Used to store settings during dev and testing */ -export const developmentSettingFolderName = 'settings-dev'; -export const developmentHttpsCertKeyFolderName = 'https-keys-dev'; +export const settingFolderName = 'settings'; +export const httpsCertKeyFolderName = 'https-keys'; +export const cacheDatabaseFolderName = 'cache-database'; /** Used to place mock wiki during dev and testing */ export const developmentWikiFolderName = 'tidgi-dev'; export const localizationFolderName = 'localization'; diff --git a/src/constants/paths.ts b/src/constants/paths.ts index 3fbc85e9..3915dc3f 100644 --- a/src/constants/paths.ts +++ b/src/constants/paths.ts @@ -27,6 +27,9 @@ export const ZX_FOLDER = isDevelopmentOrTest export const TIDDLYWIKI_PACKAGE_FOLDER = isDevelopmentOrTest ? path.resolve(__dirname, '..', '..', 'node_modules', '@tiddlygit', 'tiddlywiki', 'boot') : path.resolve(process.resourcesPath, 'node_modules', '@tiddlygit', 'tiddlywiki', 'boot'); +export const SQLITE_BINARY_PATH = isDevelopmentOrTest + ? path.resolve(__dirname, '..', '..', 'node_modules', 'better-sqlite3/build/Release/better_sqlite3.node') + : path.resolve(process.resourcesPath, 'node_modules', 'better-sqlite3/build/Release/better_sqlite3.node'); export const LOCALIZATION_FOLDER = isDevelopmentOrTest ? path.resolve(sourcePath, '..', localizationFolderName) : path.resolve(process.resourcesPath, localizationFolderName); diff --git a/src/services/database/index.ts b/src/services/database/index.ts new file mode 100644 index 00000000..62f2c136 --- /dev/null +++ b/src/services/database/index.ts @@ -0,0 +1,68 @@ +import Sqlite3Database from 'better-sqlite3'; +import { injectable } from 'inversify'; +import * as sqlite_vss from 'sqlite-vss'; + +import type { INativeService } from '@services/native/interface'; +import serviceIdentifier from '@services/serviceIdentifier'; +import { IDatabaseService } from './interface'; + +import { CACHE_DATABASE_FOLDER } from '@/constants/appPaths'; +import { lazyInject } from '@services/container'; +import { logger } from '@services/libs/log'; +import fs from 'fs-extra'; +import path from 'path'; + +@injectable() +export class DatabaseService implements IDatabaseService { + @lazyInject(serviceIdentifier.NativeService) + private readonly nativeService!: INativeService; + + // tiddlywiki require methods to be sync, so direct run them in the main process. But later we can use worker_thread to run heavier search queries, as a readonly slave db, and do some data sync between them. + // many operations has to be done in wikiWorker, so can be accessed by nodejs wiki in a sync way. + // private readonly dbWorker?: ModuleThread; + + async initializeForWorkspace(workspaceID: string): Promise { + const destinationFilePath = this.getDataBasePath(workspaceID); + // only create db file for this workspace's wiki if it doesn't exist + if (await fs.exists(this.getDataBasePath(workspaceID))) { + return; + } + await fs.ensureDir(CACHE_DATABASE_FOLDER); + try { + // create a database and table that adapts tiddlywiki usage + const database = new Sqlite3Database(':memory:', { verbose: logger.debug }); + try { + sqlite_vss.load(database); + const vssVersion = database.prepare('select vss_version()').pluck().get() as string; + logger.debug(`initializeForWorkspace using sqlite-vss version: ${vssVersion} for workspace ${workspaceID}`); + } catch (error) { + logger.error(`error when loading sqlite-vss for workspace ${workspaceID}: ${(error as Error).message}`); + } + /** + * Create table storing most commonly used tiddler fields, other fields are stored in `fields` column as a JSON string. + */ + const createTiddlywikiTable = database.prepare(` + CREATE TABLE IF NOT EXISTS tiddlers ( + title TEXT PRIMARY KEY, + text TEXT, + type TEXT, + created INTEGER, + modified INTEGER, + tags TEXT, + fields TEXT, + creator TEXT, + modifier TEXT + ); + `); + createTiddlywikiTable.run(); + await database.backup(destinationFilePath); + database.close(); + } catch (error) { + logger.error(`error when creating sqlite cache database for workspace ${workspaceID}: ${(error as Error).message}`); + } + } + + getDataBasePath(workspaceID: string): string { + return path.resolve(CACHE_DATABASE_FOLDER, `${workspaceID}-sqlite3-cache.db`); + } +} diff --git a/src/services/database/interface.ts b/src/services/database/interface.ts new file mode 100644 index 00000000..9eb8fbf6 --- /dev/null +++ b/src/services/database/interface.ts @@ -0,0 +1,21 @@ +import { DatabaseChannel } from '@/constants/channels'; +import { ProxyPropertyType } from 'electron-ipc-cat/common'; + +/** + * Allow wiki or external app to save/search tiddlers cache from database like sqlite+sqlite-vss (vector storage) + */ +export interface IDatabaseService { + getDataBasePath(workspaceID: string): string; + /** + * Create a database file for a workspace, store it in the appData folder, and load it in a worker_thread to execute SQL. * + * (not store `.db` file in the workspace wiki's folder, because this cache file shouldn't not by Database committed) + */ + initializeForWorkspace(workspaceID: string): Promise; +} +export const DatabaseServiceIPCDescriptor = { + channel: DatabaseChannel.name, + properties: { + initializeForWorkspace: ProxyPropertyType.Function, + getDataBasePath: ProxyPropertyType.Function, + }, +}; diff --git a/src/services/database/wikiWorkerOperations.ts b/src/services/database/wikiWorkerOperations.ts new file mode 100644 index 00000000..23890e55 --- /dev/null +++ b/src/services/database/wikiWorkerOperations.ts @@ -0,0 +1,58 @@ +import type { ITiddlerFields } from '@tiddlygit/tiddlywiki'; +import Sqlite3Database from 'better-sqlite3'; +import fs from 'fs-extra'; +import * as sqlite_vss from 'sqlite-vss'; + +export interface ISqliteDatabasePaths { + databaseFile: string; + sqliteBinary: string; +} +export class WikiWorkerDatabaseOperations { + #database: Sqlite3Database.Database; + constructor(paths: ISqliteDatabasePaths) { + if (!fs.existsSync(paths.databaseFile)) { + throw new SqliteDatabaseNotInitializedError(paths.databaseFile); + } + const database = new Sqlite3Database(paths.databaseFile, { verbose: console.log, fileMustExist: true, nativeBinding: paths.sqliteBinary }); + try { + sqlite_vss.load(database); + } catch { + // ignore, error already logged in src/services/database/index.ts 's `initializeForWorkspace` + } + this.#database = database; + this.prepareMethods(); + } + + insertTiddlers!: Sqlite3Database.Transaction<(tiddlers: ITiddlerFields[]) => void>; + putTiddlers(tiddlers: ITiddlerFields[]) { + this.insertTiddlers(tiddlers); + } + + private prepareMethods() { + const insertTiddler = this.#database.prepare(` + INSERT INTO tiddlers (title, text, type, created, modified, tags, fields, creator, modifier) + VALUES (@title, @text, @type, @created, @modified, @tags, @fields, @creator, @modifier) + ON CONFLICT(title) DO UPDATE SET + text = excluded.text, + type = excluded.type, + created = excluded.created, + modified = excluded.modified, + tags = excluded.tags, + fields = excluded.fields, + creator = excluded.creator, + modifier = excluded.modifier + `); + this.insertTiddlers = this.#database.transaction((tiddlers: ITiddlerFields[]) => { + for (const tiddler of tiddlers) { + insertTiddler.run(tiddler); + } + }); + } +} + +export class SqliteDatabaseNotInitializedError extends Error { + constructor(databaseFile: string) { + super(); + this.message = `database file not found (This is OK for first init of workspace, until initializeWorkspaceView call initializeForWorkspace): ${databaseFile}`; + } +} diff --git a/src/services/libs/bindServiceAndProxy.ts b/src/services/libs/bindServiceAndProxy.ts index d322beab..43e113ac 100644 --- a/src/services/libs/bindServiceAndProxy.ts +++ b/src/services/libs/bindServiceAndProxy.ts @@ -8,6 +8,7 @@ import serviceIdentifier from '@services/serviceIdentifier'; import { Authentication } from '@services/auth'; import { ContextService } from '@services/context'; +import { DatabaseService } from '@services/database'; import { Git } from '@services/git'; import { MenuService } from '@services/menu'; import { NativeService } from '@services/native'; @@ -27,6 +28,8 @@ import type { IAuthenticationService } from '@services/auth/interface'; import { AuthenticationServiceIPCDescriptor } from '@services/auth/interface'; import type { IContextService } from '@services/context/interface'; import { ContextServiceIPCDescriptor } from '@services/context/interface'; +import type { IDatabaseService } from '@services/database/interface'; +import { DatabaseServiceIPCDescriptor } from '@services/database/interface'; import type { IGitService } from '@services/git/interface'; import { GitServiceIPCDescriptor } from '@services/git/interface'; import type { IMenuService } from '@services/menu/interface'; @@ -59,6 +62,7 @@ import { WorkspaceViewServiceIPCDescriptor } from '@services/workspacesView/inte export function bindServiceAndProxy(): void { container.bind(serviceIdentifier.Authentication).to(Authentication).inSingletonScope(); container.bind(serviceIdentifier.Context).to(ContextService).inSingletonScope(); + container.bind(serviceIdentifier.Database).to(DatabaseService).inSingletonScope(); container.bind(serviceIdentifier.Git).to(Git).inSingletonScope(); container.bind(serviceIdentifier.MenuService).to(MenuService).inSingletonScope(); container.bind(serviceIdentifier.NativeService).to(NativeService).inSingletonScope(); @@ -76,6 +80,7 @@ export function bindServiceAndProxy(): void { const authService = container.get(serviceIdentifier.Authentication); const contextService = container.get(serviceIdentifier.Context); + const databaseService = container.get(serviceIdentifier.Database); const gitService = container.get(serviceIdentifier.Git); const menuService = container.get(serviceIdentifier.MenuService); const nativeService = container.get(serviceIdentifier.NativeService); @@ -93,6 +98,7 @@ export function bindServiceAndProxy(): void { registerProxy(authService, AuthenticationServiceIPCDescriptor); registerProxy(contextService, ContextServiceIPCDescriptor); + registerProxy(databaseService, DatabaseServiceIPCDescriptor); registerProxy(gitService, GitServiceIPCDescriptor); registerProxy(menuService, MenuServiceIPCDescriptor); registerProxy(nativeService, NativeServiceIPCDescriptor); diff --git a/src/services/serviceIdentifier.ts b/src/services/serviceIdentifier.ts index d622ebe3..87c7f48c 100644 --- a/src/services/serviceIdentifier.ts +++ b/src/services/serviceIdentifier.ts @@ -2,6 +2,7 @@ export default { Authentication: Symbol.for('Authentication'), Git: Symbol.for('Git'), Context: Symbol.for('Context'), + Database: Symbol.for('Database'), MenuService: Symbol.for('MenuService'), NativeService: Symbol.for('NativeService'), NotificationService: Symbol.for('NotificationService'), diff --git a/src/services/wiki/index.ts b/src/services/wiki/index.ts index 5d8b983b..5c16ca23 100644 --- a/src/services/wiki/index.ts +++ b/src/services/wiki/index.ts @@ -11,7 +11,7 @@ import { ModuleThread, spawn, Thread, Worker } from 'threads'; import type { WorkerEvent } from 'threads/dist/types/master'; import { WikiChannel } from '@/constants/channels'; -import { TIDDLERS_PATH, TIDDLYWIKI_PACKAGE_FOLDER, TIDDLYWIKI_TEMPLATE_FOLDER_PATH } from '@/constants/paths'; +import { SQLITE_BINARY_PATH, TIDDLERS_PATH, TIDDLYWIKI_PACKAGE_FOLDER, TIDDLYWIKI_TEMPLATE_FOLDER_PATH } from '@/constants/paths'; import type { IAuthenticationService } from '@services/auth/interface'; import { lazyInject } from '@services/container'; import type { IGitService, IGitUserInfos } from '@services/git/interface'; @@ -27,10 +27,11 @@ import type { IWorkspaceViewService } from '@services/workspacesView/interface'; import { CopyWikiTemplateError, DoubleWikiInstanceError, SubWikiSMainWikiNotExistError, WikiRuntimeError } from './error'; import { IWikiService, WikiControlActions } from './interface'; import { getSubWikiPluginContent, ISubWikiPluginContent, updateSubWikiPluginContent } from './plugin/subWikiPlugin'; -import type { WikiWorker } from './wikiWorker'; +import type { IStartNodeJSWikiConfigs, WikiWorker } from './wikiWorker'; import { isDevelopmentOrTest } from '@/constants/environment'; import { defaultServerIP } from '@/constants/urls'; +import { IDatabaseService } from '@services/database/interface'; import { IPreferenceService } from '@services/preferences/interface'; // @ts-expect-error it don't want .ts // eslint-disable-next-line import/no-webpack-loader-syntax @@ -45,6 +46,9 @@ export class Wiki implements IWikiService { @lazyInject(serviceIdentifier.Authentication) private readonly authService!: IAuthenticationService; + @lazyInject(serviceIdentifier.Database) + private readonly databaseService!: IDatabaseService; + @lazyInject(serviceIdentifier.Window) private readonly windowService!: IWindowService; @@ -118,7 +122,7 @@ export class Wiki implements IWikiService { adminToken = this.authService.generateOneTimeAdminAuthTokenForWorkspace(workspaceID); } } - const workerData = { + const workerData: IStartNodeJSWikiConfigs = { adminToken, constants: { TIDDLYWIKI_PACKAGE_FOLDER }, excludedPlugins, @@ -157,6 +161,15 @@ export class Wiki implements IWikiService { } }); + worker.initCacheDatabase({ + databaseFile: this.databaseService.getDataBasePath(workspaceID), + sqliteBinary: SQLITE_BINARY_PATH, + }).subscribe(async (message) => { + if (message.type === 'stderr' || message.type === 'stdout') { + wikiOutputToFile(id, message.message); + } + }); + // subscribe to the Observable that startNodeJSWiki returns, handle messages send by our code worker.startNodeJSWiki(workerData).subscribe(async (message) => { if (message.type === 'control') { @@ -575,7 +588,9 @@ export class Wiki implements IWikiService { private async syncAllSubWikiIfNeeded(workspace: IWorkspace) { const workspaces = await this.workspaceService.getWorkspacesAsList(); const subWikiWorkspaces = workspaces.filter((w) => w.mainWikiID === workspace.id); - await Promise.all(subWikiWorkspaces.map((w) => this.syncWikiIfNeeded(w))); + await Promise.all(subWikiWorkspaces.map(async (w) => { + await this.syncWikiIfNeeded(w); + })); } private stopIntervalSync(workspace: IWorkspace): void { diff --git a/src/services/wiki/wikiWorker.ts b/src/services/wiki/wikiWorker.ts index 8a3f3adf..1b75f79e 100644 --- a/src/services/wiki/wikiWorker.ts +++ b/src/services/wiki/wikiWorker.ts @@ -3,6 +3,8 @@ * Worker environment is not part of electron environment, so don't import "@/constants/paths" here, as its process.resourcesPath will become undefined and throw Errors. * * Don't use i18n and logger in worker thread. For example, 12b93020, will throw error "Electron failed to install correctly, please delete node_modules/electron and try installing again ...worker.js..." + * + * Import tw related things and typing from `@tiddlygit/tiddlywiki` instead of `tiddlywiki`, otherwise you will get `Unhandled Error ReferenceError: self is not defined at $:/boot/bootprefix.js:40749:36` because tiddlywiki */ import { uninstall } from '@/helpers/installV8Cache'; import 'source-map-support/register'; @@ -19,28 +21,17 @@ import { expose } from 'threads/worker'; import { getTidGiAuthHeaderWithToken } from '@/constants/auth'; import { isHtmlWiki } from '@/constants/fileNames'; import { defaultServerIP } from '@/constants/urls'; +import { ISqliteDatabasePaths, SqliteDatabaseNotInitializedError, WikiWorkerDatabaseOperations } from '@services/database/wikiWorkerOperations'; import { fixPath } from '@services/libs/fixPath'; -import { IWikiMessage, IZxWorkerMessage, WikiControlActions, ZxWorkerControlActions } from './interface'; +import { IWikiLogMessage, IWikiMessage, IZxWorkerMessage, WikiControlActions, ZxWorkerControlActions } from './interface'; import { executeScriptInTWContext, extractTWContextScripts, getTWVmContext } from './plugin/zxPlugin'; import { adminTokenIsProvided } from './wikiWorkerUtils'; fixPath(); let wikiInstance: ITiddlyWiki | undefined; +let cacheDatabase: WikiWorkerDatabaseOperations | undefined; -function startNodeJSWiki({ - adminToken, - constants: { TIDDLYWIKI_PACKAGE_FOLDER }, - excludedPlugins = [], - homePath, - https, - isDev, - readOnlyMode, - rootTiddler, - tiddlyWikiHost = defaultServerIP, - tiddlyWikiPort = 5112, - tokenAuth, - userName, -}: { +export interface IStartNodeJSWikiConfigs { adminToken?: string; constants: { TIDDLYWIKI_PACKAGE_FOLDER: string }; excludedPlugins: string[]; @@ -57,7 +48,38 @@ function startNodeJSWiki({ tiddlyWikiPort: number; tokenAuth?: boolean; userName: string; -}): Observable { +} + +function initCacheDatabase(cacheDatabaseConfig: ISqliteDatabasePaths) { + return new Observable((observer) => { + try { + cacheDatabase = new WikiWorkerDatabaseOperations(cacheDatabaseConfig); + } catch (error) { + if (error instanceof SqliteDatabaseNotInitializedError) { + // this is usual for first time + observer.next({ type: 'stdout', message: error.message }); + } else { + // unexpected error + observer.next({ type: 'stderr', message: (error as Error)?.message }); + } + } + }); +} + +function startNodeJSWiki({ + adminToken, + constants: { TIDDLYWIKI_PACKAGE_FOLDER }, + excludedPlugins = [], + homePath, + https, + isDev, + readOnlyMode, + rootTiddler, + tiddlyWikiHost = defaultServerIP, + tiddlyWikiPort = 5112, + tokenAuth, + userName, +}: IStartNodeJSWikiConfigs): Observable { return new Observable((observer) => { let fullBootArgv: string[] = []; observer.next({ type: 'control', actions: WikiControlActions.start, argv: fullBootArgv }); @@ -283,6 +305,7 @@ const wikiWorker = { extractWikiHTML, packetHTMLFromWikiFolder, beforeExit, + initCacheDatabase, }; export type WikiWorker = typeof wikiWorker; expose(wikiWorker); diff --git a/src/services/workspacesView/index.ts b/src/services/workspacesView/index.ts index a0bafd7e..9262e45c 100644 --- a/src/services/workspacesView/index.ts +++ b/src/services/workspacesView/index.ts @@ -12,6 +12,7 @@ import { tiddlywikiLanguagesMap } from '@/constants/languages'; import { WikiCreationMethod } from '@/constants/wikiCreation'; import type { IAuthenticationService } from '@services/auth/interface'; import { lazyInject } from '@services/container'; +import { IDatabaseService } from '@services/database/interface'; import type { IGitService } from '@services/git/interface'; import getFromRenderer from '@services/libs/getFromRenderer'; import { i18n } from '@services/libs/i18n'; @@ -41,6 +42,9 @@ export class WorkspaceView implements IWorkspaceViewService { @lazyInject(serviceIdentifier.Git) private readonly gitService!: IGitService; + @lazyInject(serviceIdentifier.Database) + private readonly databaseService!: IDatabaseService; + @lazyInject(serviceIdentifier.Wiki) private readonly wikiService!: IWikiService; @@ -203,6 +207,8 @@ export class WorkspaceView implements IWorkspaceViewService { } } } + // after all init finished, create cache database if there is no one + await this.databaseService.initializeForWorkspace(workspace.id); } public async updateLastUrl(