fix: use custom sqlite-vss node binding, to fix path issue

https://github.com/asg017/sqlite-vss/issues/41

package.json

@@ -135,7 +135,7 @@
     "css-loader": "6.7.4",
     "date-fns": "2.30.0",
     "dprint": "^0.37.1",
-    "electron": "25.0.1",
+    "electron": "24.4.1",
     "electron-wix-msi": "^5.0.0",
     "esbuild-loader": "^3.0.1",
     "eslint": "8.42.0",

patches/sqlite-vss@0.1.1-alpha.13.patch

@@ -1,13 +0,0 @@
-diff --git a/src/index.js b/src/index.js
-index 5c6b2f39fdbd05ca4b5e9e2bf6e66c9bf71489b6..825040c516711fadc889b473c129657df1a4bec2 100644
---- a/src/index.js
-+++ b/src/index.js
-@@ -34,7 +34,7 @@ function loadablePathResolver(name) {
-   }
-   const packageName = platformPackageName(platform, arch);
-   const loadablePath = join(
--    fileURLToPath(new URL(".", import.meta.url)),
-+    __dirname,
-     "..",
-     "..",
-     packageName,

pnpm-lock.yaml

@@ -21,13 +21,13 @@ dependencies:
     version: 2.5.0
   electron-ipc-cat:
     specifier: 1.2.9
-    version: 1.2.9(electron@25.0.1)(rxjs@7.8.1)
+    version: 1.2.9(electron@24.4.1)(rxjs@7.8.1)
   electron-is-packaged:
     specifier: 1.0.2
     version: 1.0.2(eslint@8.42.0)
   electron-settings:
     specifier: 5.0.0
-    version: 5.0.0(electron@25.0.1)
+    version: 5.0.0(electron@24.4.1)
   electron-squirrel-startup:
     specifier: 1.0.0
     version: 1.0.0
@@ -81,7 +81,7 @@ dependencies:
     version: 4.17.21
   menubar:
     specifier: 9.3.0
-    version: 9.3.0(electron@25.0.1)
+    version: 9.3.0(electron@24.4.1)
   nanoid:
     specifier: ^4.0.2
     version: 4.0.2
@@ -318,8 +318,8 @@ devDependencies:
     specifier: ^0.37.1
     version: 0.37.1
   electron:
-    specifier: 25.0.1
-    version: 25.0.1
+    specifier: 24.4.1
+    version: 24.4.1
   electron-wix-msi:
     specifier: ^5.0.0
     version: 5.0.0
@@ -4985,13 +4985,13 @@ packages:
     dev: true
     optional: true
 
-  /electron-ipc-cat@1.2.9(electron@25.0.1)(rxjs@7.8.1):
+  /electron-ipc-cat@1.2.9(electron@24.4.1)(rxjs@7.8.1):
     resolution: {integrity: sha512-H/d2/sDlc4BP7IlVUONWipGLxYzRxQOVhO5So/DQ7kYCOm4hW+PdHFE0EKUzNeK5W9hQaiYAuQ22k9AQJ/VAyA==}
     peerDependencies:
       electron: '>= 13.0.0'
       rxjs: '>= 7.5.0'
     dependencies:
-      electron: 25.0.1
+      electron: 24.4.1
       errio: 1.2.2
       lodash: 4.17.21
       rxjs: 7.8.1
@@ -5046,13 +5046,13 @@ packages:
     resolution: {integrity: sha512-726DfbI9ZNoCg+Fcu6XLuTKTnzf+6nFqv7h+K/V6Ug7IbaPMI7s9S8URnGtWFCy5N5PL4HSzRFF2mXuinftDdg==}
     dev: false
 
-  /electron-settings@5.0.0(electron@25.0.1):
+  /electron-settings@5.0.0(electron@24.4.1):
     resolution: {integrity: sha512-QZOtjW3mPmyDE7VM4yHPS2JCmExwqaqPrArldgOdOLz25/yrn8qXPB5ZkiqPeCTXjdB7TZc4BFbOMjUBEbatSA==}
     peerDependencies:
       electron: '>= 2'
     dependencies:
       atomically: 2.0.1
-      electron: 25.0.1
+      electron: 24.4.1
       lodash.get: 4.4.2
       lodash.has: 4.5.2
       lodash.set: 4.3.2
@@ -5130,8 +5130,8 @@ packages:
       - supports-color
     dev: true
 
-  /electron@25.0.1:
-    resolution: {integrity: sha512-YD3xCrH01LiPeLlG90DWgMXJK69UxY4NiXKqXT12HOiXLqEaKrLWap+CiiS7J7SWUXz+4XOItQI8g1dtG7zkkA==}
+  /electron@24.4.1:
+    resolution: {integrity: sha512-tVZw5/dKGJbU1h8bdY0gfz4Y7ANYi8VddnIiLg/sPidJWc4CdshCZhLfU0lW1ySjtEt35JYxGHfmerkJJqUSTw==}
     engines: {node: '>= 12.20.55'}
     hasBin: true
     requiresBuild: true
@@ -8001,12 +8001,12 @@ packages:
       fs-monkey: 1.0.3
     dev: true
 
-  /menubar@9.3.0(electron@25.0.1):
+  /menubar@9.3.0(electron@24.4.1):
     resolution: {integrity: sha512-TC7R5YR+OKVfFIyGiZ+HkKmY0hMLhOseX6wN0DxWzzF1CNhZ2Tm3hMB6erc35jzBmrsMDghRm89dtSeY0VjmjQ==}
     peerDependencies:
       electron: ^9.0.0 || ^10.0.0 || ^11.0.0 || ^12.0.0 || ^13.0.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^20.0.0 || ^21.0.0 || ^22.0.0
     dependencies:
-      electron: 25.0.1
+      electron: 24.4.1
       electron-positioner: 4.1.0
     dev: false
 

scripts/afterPack.js

@@ -44,7 +44,7 @@ exports.default = async (buildPath, electronVersion, platform, arch, callback) =
     await fs.mkdirp(path.join(cwd, 'node_modules', 'app-path'));
     await fs.copy(path.join(projectRoot, 'node_modules', 'app-path', 'main'), path.join(cwd, 'node_modules', 'app-path', 'main'), { dereference: true });
     await fs.copy(path.resolve(projectRoot, 'node_modules/better-sqlite3/build/Release/better_sqlite3.node'), path.resolve(cwd, 'node_modules/better-sqlite3/build/Release/better_sqlite3.node'), { dereference: true });
-    const sqliteVssPackages = ['sqlite-vss', 'sqlite-vss-linux-x64', 'sqlite-vss-darwin-x64', 'sqlite-vss-darwin-arm64']
+    const sqliteVssPackages = ['sqlite-vss-linux-x64', 'sqlite-vss-darwin-x64', 'sqlite-vss-darwin-arm64']
     for (const sqliteVssPackage of sqliteVssPackages) {
       try {
         await fs.copy(path.resolve(projectRoot, `node_modules/${sqliteVssPackage}`), path.resolve(cwd, `node_modules/${sqliteVssPackage}`), { dereference: true });

src/constants/paths.ts

@@ -21,15 +21,14 @@ export const MENUBAR_ICON_PATH = path.resolve(isDevelopmentOrTest ? buildResourc
 export const CHROME_ERROR_PATH = 'chrome-error://chromewebdata/';
 export const LOGIN_REDIRECT_PATH = 'http://localhost:3012/?code=';
 export const DESKTOP_PATH = path.join(os.homedir(), 'Desktop');
-export const ZX_FOLDER = isDevelopmentOrTest
-  ? path.resolve(__dirname, '..', '..', 'node_modules', 'zx', 'build', 'cli.js')
-  : path.resolve(process.resourcesPath, 'node_modules', 'zx', 'build', 'cli.js');
-export const TIDDLYWIKI_PACKAGE_FOLDER = isDevelopmentOrTest
-  ? path.resolve(__dirname, '..', '..', 'node_modules', '@tiddlygit', 'tiddlywiki', 'boot')
-  : path.resolve(process.resourcesPath, 'node_modules', '@tiddlygit', 'tiddlywiki', 'boot');
-export const SQLITE_BINARY_PATH = isDevelopmentOrTest
-  ? path.resolve(__dirname, '..', '..', 'node_modules', 'better-sqlite3/build/Release/better_sqlite3.node')
-  : path.resolve(process.resourcesPath, 'node_modules', 'better-sqlite3/build/Release/better_sqlite3.node');
+
+export const PACKAGE_PATH_BASE = isDevelopmentOrTest
+  ? path.resolve(__dirname, '..', '..', 'node_modules')
+  : path.resolve(process.resourcesPath, 'node_modules');
+export const ZX_FOLDER = path.resolve(PACKAGE_PATH_BASE, 'zx', 'build', 'cli.js');
+export const TIDDLYWIKI_PACKAGE_FOLDER = path.resolve(PACKAGE_PATH_BASE, '@tiddlygit', 'tiddlywiki', 'boot');
+export const SQLITE_BINARY_PATH = path.resolve(PACKAGE_PATH_BASE, 'better-sqlite3', 'build', 'Release', 'better_sqlite3.node');
+
 export const LOCALIZATION_FOLDER = isDevelopmentOrTest
   ? path.resolve(sourcePath, '..', localizationFolderName)
   : path.resolve(process.resourcesPath, localizationFolderName);

src/services/database/index.ts

@@ -1,16 +1,17 @@
 import Sqlite3Database from 'better-sqlite3';
 import { injectable } from 'inversify';
-import * as sqlite_vss from 'sqlite-vss';
 
 import type { INativeService } from '@services/native/interface';
 import serviceIdentifier from '@services/serviceIdentifier';
 import { IDatabaseService } from './interface';
 
 import { CACHE_DATABASE_FOLDER } from '@/constants/appPaths';
+import { PACKAGE_PATH_BASE, SQLITE_BINARY_PATH } from '@/constants/paths';
 import { lazyInject } from '@services/container';
 import { logger } from '@services/libs/log';
 import fs from 'fs-extra';
 import path from 'path';
+import { loadSqliteVss } from './sqlite-vss';
 
 @injectable()
 export class DatabaseService implements IDatabaseService {
@@ -25,14 +26,16 @@ export class DatabaseService implements IDatabaseService {
     const destinationFilePath = this.getDataBasePath(workspaceID);
     // only create db file for this workspace's wiki if it doesn't exist
     if (await fs.exists(this.getDataBasePath(workspaceID))) {
+      logger.debug(`initializeForWorkspace skip, there already has sqlite database for workspace ${workspaceID} in ${destinationFilePath}`);
       return;
     }
     await fs.ensureDir(CACHE_DATABASE_FOLDER);
     try {
       // create a database and table that adapts tiddlywiki usage
-      const database = new Sqlite3Database(':memory:', { verbose: logger.debug });
+      logger.debug(`initializeForWorkspace create a sqlite database with vss and table that adapts tiddlywiki usage for workspace ${workspaceID}`);
+      const database = new Sqlite3Database(':memory:', { verbose: logger.debug, nativeBinding: SQLITE_BINARY_PATH });
       try {
-        sqlite_vss.load(database);
+        loadSqliteVss(database, PACKAGE_PATH_BASE);
         const vssVersion = database.prepare('select vss_version()').pluck().get() as string;
         logger.debug(`initializeForWorkspace using sqlite-vss version: ${vssVersion} for workspace ${workspaceID}`);
       } catch (error) {

src/services/database/sqlite-vss/index.ts

@@ -0,0 +1,72 @@
+import type Sqlite3Database from 'better-sqlite3';
+import fs from 'fs-extra';
+import path from 'path';
+import { arch, platform } from 'process';
+
+const supportedPlatforms: Array<[string, string]> = [
+  ['darwin', 'x64'],
+  ['darwin', 'arm64'],
+  ['linux', 'x64'],
+];
+
+function validPlatform(platform: string, arch: string): boolean {
+  return supportedPlatforms.find(([p, a]) => platform === p && arch === a) !== null;
+}
+
+function extensionSuffix(platform: string): string {
+  if (platform === 'win32') return 'dll';
+  if (platform === 'darwin') return 'dylib';
+  return 'so';
+}
+
+function platformPackageName(platform: string, arch: string): string {
+  const os = platform === 'win32' ? 'windows' : platform;
+  return `sqlite-vss-${os}-${arch}`;
+}
+
+function loadablePathResolver(name: string, PACKAGE_PATH_BASE: string): string {
+  if (!validPlatform(platform, arch)) {
+    throw new Error(
+      `Unsupported platform for sqlite-vss, on a ${platform}-${arch} machine, but not in supported platforms (${
+        supportedPlatforms
+          .map(([p, a]) => `${p}-${a}`)
+          .join(',')
+      }). Consult the sqlite-vss NPM package README for details. `,
+    );
+  }
+  const packageName = platformPackageName(platform, arch);
+  const loadablePath = path.join(
+    PACKAGE_PATH_BASE,
+    packageName,
+    'lib',
+    `${name}.${extensionSuffix(platform)}`,
+  );
+  if (fs.statSync(loadablePath, { throwIfNoEntry: false }) === undefined) {
+    throw new Error(
+      `Loadble extension for sqlite-vss not found in ${loadablePath}. Was the ${packageName} package installed? Avoid using the --no-optional flag, as the optional dependencies for sqlite-vss are required.`,
+    );
+  }
+
+  return loadablePath;
+}
+
+export function getVectorLoadablePath(PACKAGE_PATH_BASE: string): string {
+  return loadablePathResolver('vector0', PACKAGE_PATH_BASE);
+}
+
+export function getVssLoadablePath(PACKAGE_PATH_BASE: string): string {
+  return loadablePathResolver('vss0', PACKAGE_PATH_BASE);
+}
+
+export function loadVector(database: Sqlite3Database.Database, PACKAGE_PATH_BASE: string): void {
+  database.loadExtension(getVectorLoadablePath(PACKAGE_PATH_BASE));
+}
+
+export function loadVss(database: Sqlite3Database.Database, PACKAGE_PATH_BASE: string): void {
+  database.loadExtension(getVssLoadablePath(PACKAGE_PATH_BASE));
+}
+
+export function loadSqliteVss(database: Sqlite3Database.Database, PACKAGE_PATH_BASE: string): void {
+  loadVector(database, PACKAGE_PATH_BASE);
+  loadVss(database, PACKAGE_PATH_BASE);
+}

src/services/database/wikiWorkerOperations.ts

@@ -1,10 +1,11 @@
 import type { ITiddlerFields } from '@tiddlygit/tiddlywiki';
 import Sqlite3Database from 'better-sqlite3';
 import fs from 'fs-extra';
-import * as sqlite_vss from 'sqlite-vss';
+import { loadSqliteVss } from './sqlite-vss';
 
 export interface ISqliteDatabasePaths {
   databaseFile: string;
+  packagePathBase: string;
   sqliteBinary: string;
 }
 export class WikiWorkerDatabaseOperations {
@@ -14,13 +15,17 @@ export class WikiWorkerDatabaseOperations {
       throw new SqliteDatabaseNotInitializedError(paths.databaseFile);
     }
     const database = new Sqlite3Database(paths.databaseFile, { verbose: console.log, fileMustExist: true, nativeBinding: paths.sqliteBinary });
+    this.#database = database;
+    this.prepareMethods();
+    this.loadVSS(database, paths);
+  }
+
+  loadVSS(database: Sqlite3Database.Database, paths: ISqliteDatabasePaths) {
     try {
-      sqlite_vss.load(database);
+      loadSqliteVss(database, paths.packagePathBase);
     } catch {
       // ignore, error already logged in src/services/database/index.ts 's `initializeForWorkspace`
     }
-    this.#database = database;
-    this.prepareMethods();
   }
 
   insertTiddlers!: Sqlite3Database.Transaction<(tiddlers: ITiddlerFields[]) => void>;

src/services/wiki/index.ts

@@ -11,7 +11,7 @@ import { ModuleThread, spawn, Thread, Worker } from 'threads';
 import type { WorkerEvent } from 'threads/dist/types/master';
 
 import { WikiChannel } from '@/constants/channels';
-import { SQLITE_BINARY_PATH, TIDDLERS_PATH, TIDDLYWIKI_PACKAGE_FOLDER, TIDDLYWIKI_TEMPLATE_FOLDER_PATH } from '@/constants/paths';
+import { PACKAGE_PATH_BASE, SQLITE_BINARY_PATH, TIDDLERS_PATH, TIDDLYWIKI_PACKAGE_FOLDER, TIDDLYWIKI_TEMPLATE_FOLDER_PATH } from '@/constants/paths';
 import type { IAuthenticationService } from '@services/auth/interface';
 import { lazyInject } from '@services/container';
 import type { IGitService, IGitUserInfos } from '@services/git/interface';
@@ -161,6 +161,7 @@ export class Wiki implements IWikiService {
       worker.initCacheDatabase({
         databaseFile: this.databaseService.getDataBasePath(workspaceID),
         sqliteBinary: SQLITE_BINARY_PATH,
+        packagePathBase: PACKAGE_PATH_BASE,
       }).subscribe(async (message) => {
         if (message.type === 'stderr' || message.type === 'stdout') {
           wikiOutputToFile(id, message.message);