mirror of
https://github.com/tiddly-gittly/TidGi-Desktop.git
synced 2025-12-05 18:20:39 -08:00
* Add watch-filesystem-adaptor plugin and worker IPC Introduces the watch-filesystem-adaptor TiddlyWiki plugin, enabling tag-based routing of tiddlers to sub-wikis by querying workspace info via worker thread IPC. Adds workerServiceCaller utility for worker-to-main service calls, updates workerAdapter and bindServiceAndProxy to support explicit service registration for workers, and documents the new IPC architecture. Updates wikiWorker and startNodeJSWiki to preload workspace ID and load the new plugin. Also updates the plugin build script to compile and copy the new plugin. * test: wiki operation steps * Add per-wiki labeled logging and console hijack Introduces labeled loggers for each wiki, writing logs to separate files. Adds a logFor method to NativeService for logging with labels, updates interfaces, and hijacks worker thread console methods to redirect logs to main process for wiki-specific logging. Refactors workspaceID usage to workspace object for improved context. * Update log handling for wiki worker and tests Enhanced logging tests to check all log files, including wiki logs. Adjusted logger to write wiki worker logs to the main log directory. Updated e2e app script comment for correct usage. * Enable worker thread access to main process services Introduces a proxy system allowing worker threads to call main process services with full type safety and observable support. Adds worker-side service proxy creation, auto-attaches proxies to global.service, and updates service registration to use IPC descriptors. Documentation is added for usage and architecture. * Update ErrorDuringStart.md * chore: upgrade ipc cat and allow clean vite cache * Refactor wiki worker initialization and service readiness Moved wiki worker implementation from wikiWorker.ts to wikiWorker/index.ts and deleted the old file. Added servicesReady.ts to manage worker service readiness and callbacks, and integrated notifyServicesReady into the worker lifecycle. Updated console hijack logic to wait for service readiness before hijacking. Improved worker management in Wiki service to support detaching workers and notifying readiness. * Refactor wiki logging to use centralized logger Removed per-wiki loggers and console hijacking in favor of a single labeled logger. All wiki logs, including errors, are now written to a unified log file. Updated worker and service code to route logs through the main logger and removed obsolete log file naming and management logic. * fix: ipc cat log error * Refactor wiki test paths and improve file save logic Updated test step to use wikiTestRootPath for directory replacements and added wikiTestRootPath to paths.ts for clarity. Improved error handling and directory logic in watch-filesystem-adaptor.ts, including saving tiddlers directly to sub-wiki folders, more informative logging, and ensuring cleanup after file writes is properly awaited. * rename * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * feat: basic watch-fs * feat: check file not exist * refactor: use exponential-backoff * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * Initial commit when init a new git. * fix: cleanup * Refactor test setup and cleanup to separate file Moved Before and After hooks from application.ts to a new cleanup.ts file for better organization and separation of concerns. Also removed unused imports and related code from application.ts. Minor type simplification in agent.ts for row parsing. * test: modify and rename * feat: enableFileSystemWatch * refactor: unused utils.ts * Update watch-filesystem-adaptor.ts * refactor: use node-sentinel-file-watcher * refactor: extract to two classes * The logFor method lacks JSDoc describing the level parameter's * Update startNodeJSWiki.ts * fix: napi build * Update electron-rebuild command in workflows Changed the electron-rebuild command in release and test GitHub Actions workflows to use a comma-separated list for native modules instead of multiple -w flags. This simplifies the rebuild step for better-sqlite3 and nsfw modules. * lint * not build nsfw, try use prebuild * Update package.json * Update workerAdapter.ts * remove subWikiPlugin.ts as we use new filesystem adaptor that supports tag based sub wiki * fix: build * fix: wrong type * lint * remove `act(...)` warnings * uninstall chokidar * refactor and test * lint * remove unused logic, as we already use ipc syncadaptor, not afriad of wiki status change * Update translation.json * test: increast timeout in CI * Update application.ts * fix: AI's wrong cleanup logic hidden by as unknown as * fix: AI's wrong as unknown as * Update agent.feature * Update wikiSearchPlugin.ts * fix: A dynamic import callback was not specified.
255 lines
12 KiB
TypeScript
255 lines
12 KiB
TypeScript
/**
|
|
* Expose apis that ipc sync adaptor will use
|
|
*/
|
|
|
|
import fs from 'fs-extra';
|
|
import omit from 'lodash/omit';
|
|
import path from 'path';
|
|
import { Observable } from 'rxjs';
|
|
import type { IChangedTiddlers, ITiddlerFields, ITiddlyWiki, OutputMimeTypes } from 'tiddlywiki';
|
|
|
|
export interface IWikiServerStatusObject {
|
|
anonymous: boolean;
|
|
read_only: boolean;
|
|
space: {
|
|
recipe: string;
|
|
};
|
|
tiddlywiki_version: string;
|
|
username: string;
|
|
}
|
|
export interface IWikiServerRouteResponse {
|
|
data?: string | Buffer | Array<Omit<ITiddlerFields, 'text'>> | IWikiServerStatusObject | ITiddlerFields;
|
|
headers?: Record<string, string>;
|
|
statusCode?: number;
|
|
}
|
|
|
|
export class IpcServerRoutes {
|
|
private wikiInstance!: ITiddlyWiki;
|
|
private readonly pendingIpcServerRoutesRequests: Array<(value: void | PromiseLike<void>) => void> = [];
|
|
#readonlyMode = false;
|
|
|
|
setConfig({ readOnlyMode }: { readOnlyMode?: boolean }) {
|
|
this.#readonlyMode = Boolean(readOnlyMode);
|
|
}
|
|
|
|
setWikiInstance(wikiInstance: ITiddlyWiki) {
|
|
this.wikiInstance = wikiInstance;
|
|
this.pendingIpcServerRoutesRequests.forEach((resolve) => {
|
|
resolve();
|
|
});
|
|
}
|
|
|
|
private async waitForIpcServerRoutesAvailable() {
|
|
if (this.wikiInstance !== undefined) {
|
|
return;
|
|
}
|
|
await new Promise<void>((resolve) => {
|
|
this.pendingIpcServerRoutesRequests.push(resolve);
|
|
});
|
|
}
|
|
|
|
// ████████ ██ ██████ ██████ ██ ██ ██ ██ ██ ███████ ██████
|
|
// ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██
|
|
// ██ ██ ██ ██ ██ ██ ██ ████ █████ ██ █ ██ █████ ██████
|
|
// ██ ██ ██ ██ ██ ██ ██ ██ ██ ███ ██ ██ ██ ██
|
|
// ██ ██ ██████ ██████ ███████ ██ ███ ███ ███████ ██████
|
|
|
|
async deleteTiddler(title: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
this.wikiInstance.wiki.deleteTiddler(title);
|
|
return { headers: { 'Content-Type': 'text/plain' }, data: 'OK', statusCode: 204 };
|
|
}
|
|
|
|
async getFavicon(): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const buffer = this.wikiInstance.wiki.getTiddlerText('$:/favicon.ico', '');
|
|
return { headers: { 'Content-Type': 'image/x-icon; charset=base64' }, data: buffer, statusCode: 200 };
|
|
}
|
|
|
|
async getFile(suppliedFilename: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
if (this.wikiInstance.boot.wikiPath === undefined) {
|
|
return { statusCode: 404, headers: { 'Content-Type': 'text/plain' }, data: `$tw.wiki.boot.wikiPath === undefined.` };
|
|
}
|
|
const baseFilename = path.resolve(this.wikiInstance.boot.wikiPath, 'files');
|
|
const filename = path.resolve(baseFilename, suppliedFilename);
|
|
const extension = path.extname(filename);
|
|
if (path.relative(baseFilename, filename).indexOf('..') === 0) {
|
|
return { statusCode: 404, headers: { 'Content-Type': 'text/plain' }, data: `File ${suppliedFilename} not found` };
|
|
} else {
|
|
// Send the file
|
|
try {
|
|
const data = await fs.readFile(filename);
|
|
|
|
const type = this.wikiInstance.config.fileExtensionInfo[extension] ? this.wikiInstance.config.fileExtensionInfo[extension].type : 'application/octet-stream';
|
|
return ({ statusCode: 200, headers: { 'Content-Type': type }, data });
|
|
} catch (error) {
|
|
return { statusCode: 404, headers: { 'Content-Type': 'text/plain' }, data: `Error accessing file ${suppliedFilename} with error: ${(error as Error).toString()}` };
|
|
}
|
|
}
|
|
}
|
|
|
|
async getIndex(rootTiddler: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const wikiHTML = this.wikiInstance.wiki.renderTiddler('text/plain', rootTiddler);
|
|
return { statusCode: 200, headers: { 'Content-Type': 'text/html' }, data: wikiHTML };
|
|
}
|
|
|
|
async getStatus(userName: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const data: IWikiServerStatusObject = {
|
|
username: userName,
|
|
anonymous: false,
|
|
read_only: this.#readonlyMode,
|
|
space: {
|
|
recipe: 'default',
|
|
},
|
|
tiddlywiki_version: this.wikiInstance.version,
|
|
};
|
|
return { statusCode: 200, headers: { 'Content-Type': 'application/json' }, data };
|
|
}
|
|
|
|
async getTiddler(title: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const tiddler = this.wikiInstance.wiki.getTiddler(title);
|
|
if (tiddler === undefined) {
|
|
return { statusCode: 404, headers: { 'Content-Type': 'text/plain' }, data: `Tiddler "${title}" not exist` };
|
|
}
|
|
|
|
const tiddlerFields = { ...tiddler.fields };
|
|
|
|
// only add revision if it > 0 or exists
|
|
|
|
if (this.wikiInstance.wiki.getChangeCount(title)) {
|
|
tiddlerFields.revision = String(this.wikiInstance.wiki.getChangeCount(title));
|
|
}
|
|
tiddlerFields.bag = 'default';
|
|
tiddlerFields.type = tiddlerFields.type ?? 'text/vnd.tiddlywiki';
|
|
return { statusCode: 200, headers: { 'Content-Type': 'application/json; charset=utf8' }, data: tiddlerFields as ITiddlerFields };
|
|
}
|
|
|
|
async getTiddlersJSON(
|
|
filter = '[all[tiddlers]!is[system]sort[title]]',
|
|
excludeFields = ['text'],
|
|
options?: { ignoreSyncSystemConfig?: boolean; toTiddler?: boolean },
|
|
): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
if (!(options?.ignoreSyncSystemConfig === true) && this.wikiInstance.wiki.getTiddlerText('$:/config/SyncSystemTiddlersFromServer') !== 'yes') {
|
|
filter += '+[!is[system]]';
|
|
}
|
|
const titles = this.wikiInstance.wiki.filterTiddlers(filter);
|
|
const tiddlers: Array<Omit<ITiddlerFields, 'text'>> = options?.toTiddler === false
|
|
? titles.map(title => ({ title }))
|
|
: titles.map(title => {
|
|
const tiddler = this.wikiInstance.wiki.getTiddler(title);
|
|
if (tiddler === undefined) {
|
|
return tiddler;
|
|
}
|
|
const tiddlerFields = omit(tiddler.fields, excludeFields) as Record<string, string | number>;
|
|
// only add revision if it > 0 or exists
|
|
|
|
if (this.wikiInstance.wiki.getChangeCount(title)) {
|
|
tiddlerFields.revision = String(this.wikiInstance.wiki.getChangeCount(title));
|
|
}
|
|
tiddlerFields.type = tiddlerFields.type ?? 'text/vnd.tiddlywiki';
|
|
return tiddlerFields as Omit<ITiddlerFields, 'text'>;
|
|
})
|
|
.filter((item): item is Omit<ITiddlerFields, 'text'> => item !== undefined);
|
|
return { statusCode: 200, headers: { 'Content-Type': 'application/json' }, data: tiddlers };
|
|
}
|
|
|
|
async putTiddler(title: string, fields: ITiddlerFields): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const tiddlerFieldsToPut = omit(fields, ['fields', 'revision', '_is_skinny']) as Record<string, string | number>;
|
|
// Pull up any subfields in the `fields` object
|
|
// we skip this part by not creating the fields object at the beginning at src\services\wiki\plugin\ipcSyncAdaptor\index.ts
|
|
// if ('fields' in fields) {
|
|
// tiddlerFieldsToPut = {
|
|
// ...fields.fields as Record<string, string | number>,
|
|
// };
|
|
// }
|
|
// tiddlerFieldsToPut = {
|
|
// ...tiddlerFieldsToPut,
|
|
// // Remove any revision field
|
|
// ...omit(fields, ['fields', 'revision', '_is_skinny']) as Record<string, string | number>,
|
|
// };
|
|
// If this is a skinny tiddler, it means the client never got the full
|
|
// version of the tiddler to edit. So we must preserve whatever text
|
|
// already exists on the server, or else we'll inadvertently delete it.
|
|
if (fields._is_skinny !== undefined) {
|
|
const tiddler = this.wikiInstance.wiki.getTiddler(title);
|
|
if (tiddler !== undefined) {
|
|
tiddlerFieldsToPut.text = tiddler.fields.text;
|
|
}
|
|
}
|
|
tiddlerFieldsToPut.title = title;
|
|
this.wikiInstance.wiki.addTiddler(new this.wikiInstance.Tiddler(tiddlerFieldsToPut));
|
|
const changeCount = this.wikiInstance.wiki.getChangeCount(title).toString();
|
|
return { statusCode: 204, headers: { 'Content-Type': 'text/plain', Etag: `"default/${encodeURIComponent(title)}/${changeCount}:"` }, data: 'OK' };
|
|
}
|
|
|
|
async getTiddlerHtml(title: string): Promise<IWikiServerRouteResponse> {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
const tiddler = this.wikiInstance.wiki.getTiddler(title);
|
|
if (tiddler === undefined) {
|
|
return { statusCode: 404, headers: { 'Content-Type': 'text/plain' }, data: `Tiddler "${title}" not exist` };
|
|
} else {
|
|
let renderType: OutputMimeTypes = tiddler.getFieldString('_render_type') as OutputMimeTypes;
|
|
let renderTemplate: string = tiddler.getFieldString('_render_template');
|
|
// Tiddler fields '_render_type' and '_render_template' overwrite
|
|
// system wide settings for render type and template
|
|
if (this.wikiInstance.wiki.isSystemTiddler(title)) {
|
|
renderType = renderType ?? /* this.wikiInstance.server.get('system-tiddler-render-type') ?? */ 'text/plain';
|
|
renderTemplate = renderTemplate ?? /* this.wikiInstance.server.get('system-tiddler-render-template') ?? */ '$:/core/templates/wikified-tiddler';
|
|
} else {
|
|
renderType = renderType ?? /* this.wikiInstance.server.get('tiddler-render-type') ?? */ 'text/html';
|
|
renderTemplate = renderTemplate ?? /* this.wikiInstance.server.get('tiddler-render-template') ?? */ '$:/core/templates/server/static.tiddler.html';
|
|
}
|
|
const text = this.wikiInstance.wiki.renderTiddler(renderType, renderTemplate, { parseAsInline: true, variables: { currentTiddler: title } });
|
|
|
|
// Naughty not to set a content-type, but it's the easiest way to ensure the browser will see HTML pages as HTML, and accept plain text tiddlers as CSS or JS
|
|
return { statusCode: 200, headers: { 'Content-Type': '; charset=utf8' }, data: text };
|
|
}
|
|
}
|
|
|
|
// ████████ ██ ██ ███████ ███████ ███████
|
|
// ██ ██ ██ ██ ██ ██
|
|
// ██ ██ █ ██ █████ ███████ ███████ █████
|
|
// ██ ██ ███ ██ ██ ██ ██
|
|
// ██ ███ ███ ███████ ███████ ███████
|
|
getWikiChangeObserver() {
|
|
return new Observable<IChangedTiddlers>((observer) => {
|
|
const getWikiChangeObserverInWorkerIIFE = async () => {
|
|
await this.waitForIpcServerRoutesAvailable();
|
|
if (this.wikiInstance === undefined) {
|
|
observer.error(new Error(`this.wikiInstance is undefined, maybe something went wrong between waitForIpcServerRoutesAvailable and return new Observable.`));
|
|
}
|
|
this.wikiInstance.wiki.addEventListener('change', (changes) => {
|
|
observer.next(changes);
|
|
});
|
|
console.log('[test-id-SSE_READY] Wiki change observer registered and ready');
|
|
};
|
|
void getWikiChangeObserverInWorkerIIFE();
|
|
});
|
|
}
|
|
}
|
|
|
|
export const ipcServerRoutes: IpcServerRoutes = new IpcServerRoutes();
|
|
export const ipcServerRoutesMethods = {
|
|
deleteTiddler: ipcServerRoutes.deleteTiddler.bind(ipcServerRoutes),
|
|
getFavicon: ipcServerRoutes.getFavicon.bind(ipcServerRoutes),
|
|
getIndex: ipcServerRoutes.getIndex.bind(ipcServerRoutes),
|
|
getStatus: ipcServerRoutes.getStatus.bind(ipcServerRoutes),
|
|
getTiddler: ipcServerRoutes.getTiddler.bind(ipcServerRoutes),
|
|
getTiddlerHtml: ipcServerRoutes.getTiddlerHtml.bind(ipcServerRoutes),
|
|
getTiddlersJSON: ipcServerRoutes.getTiddlersJSON.bind(ipcServerRoutes),
|
|
putTiddler: ipcServerRoutes.putTiddler.bind(ipcServerRoutes),
|
|
getFile: ipcServerRoutes.getFile.bind(ipcServerRoutes),
|
|
getWikiChangeObserver: ipcServerRoutes.getWikiChangeObserver.bind(ipcServerRoutes),
|
|
};
|
|
|
|
/**
|
|
* Available methods for ipcServerRoutes exposed from wiki worker
|
|
*/
|
|
export type IpcServerRouteMethods = Omit<typeof ipcServerRoutesMethods, 'getWikiChangeObserver'>;
|
|
export type IpcServerRouteNames = keyof IpcServerRouteMethods;
|