fix: some wikiWorker log is very long, cause lagging

src/constants/logger.ts

@@ -0,0 +1,11 @@
+export const levels = {
+  emerg: 0,
+  alert: 1,
+  crit: 2,
+  error: 3,
+  warning: 4,
+  warn: 5,
+  notice: 6,
+  info: 7,
+  debug: 8,
+};

src/services/libs/log/index.ts

@@ -2,20 +2,10 @@ import { LOG_FOLDER } from '@/constants/appPaths';
 import winston, { format } from 'winston';
 import RendererTransport from './rendererTransport';
 import 'winston-daily-rotate-file';
+import { levels } from '@/constants/logger';
 
 export * from './wikiOutput';
 
-const levels = {
-  emerg: 0,
-  alert: 1,
-  crit: 2,
-  error: 3,
-  warning: 4,
-  warn: 5,
-  notice: 6,
-  info: 7,
-  debug: 8,
-};
 export type ILogLevels = keyof typeof levels;
 const logger = (
   process.env.NODE_ENV === 'test'

src/services/libs/log/wikiOutput.ts

@@ -1,34 +1,73 @@
-import fs from 'fs-extra';
-import path from 'path';
-
 import { LOG_FOLDER } from '@/constants/appPaths';
-import { logger } from '.';
+import winston, { format } from 'winston';
+import 'winston-daily-rotate-file';
+import { levels } from '@/constants/logger';
 
-export function getWikiLogFilePath(wikiName: string): string {
+function getWikiLogFileName(workspaceID: string, wikiName: string): string {
   const logFileName = wikiName.replaceAll(/["*/:<>?\\|]/g, '_');
-  const logFilePath = path.join(LOG_FOLDER, `${logFileName}.log`);
-  return logFilePath;
-}
-export async function wikiOutputToFile(wikiName: string, message: string) {
-  try {
-    await fs.appendFile(getWikiLogFilePath(wikiName), message);
-  } catch (error) {
-    logger.error(`${getWikiLogFilePath(wikiName)}: ${(error as Error).message})}`, { function: 'wikiOutputToFile' });
-  }
+  return `${workspaceID}-${logFileName}.log`;
 }
 
+const wikiLoggers: Record<string, winston.Logger> = {};
+
 /**
- * Recreate log file
+ * Create log file using winston
  * @param {string} wikiName
  */
-export async function refreshOutputFile(wikiName: string) {
-  try {
-    const logFilePath = getWikiLogFilePath(wikiName);
-    if (await fs.exists(logFilePath)) {
-      await fs.remove(logFilePath);
-    }
-    await fs.createFile(logFilePath);
-  } catch (error) {
-    logger.error(`${getWikiLogFilePath(wikiName)}: ${(error as Error).message})}`, { function: 'refreshOutputFile' });
+export function startWikiLogger(workspaceID: string, wikiName: string) {
+  if (getWikiLogger(workspaceID) !== undefined) {
+    stopWikiLogger(workspaceID);
   }
+  const wikiLogger = (
+    process.env.NODE_ENV === 'test'
+      ? Object.assign(console, {
+        emerg: console.error.bind(console),
+        alert: console.error.bind(console),
+        crit: console.error.bind(console),
+        warning: console.warn.bind(console),
+        notice: console.log.bind(console),
+        debug: console.log.bind(console),
+        close: () => {},
+      })
+      : winston.createLogger({
+        levels,
+        transports: [
+          new winston.transports.Console(),
+          new winston.transports.DailyRotateFile({
+            filename: getWikiLogFileName(workspaceID, wikiName),
+            datePattern: 'YYYY-MM-DD',
+            zippedArchive: false,
+            maxSize: '20mb',
+            maxFiles: '14d',
+            dirname: LOG_FOLDER,
+            level: 'debug',
+          }),
+        ],
+        exceptionHandlers: [
+          new winston.transports.DailyRotateFile({
+            filename: `error-${getWikiLogFileName(workspaceID, wikiName)}`,
+            datePattern: 'YYYY-MM-DD',
+            zippedArchive: false,
+            maxSize: '20mb',
+            maxFiles: '14d',
+            dirname: LOG_FOLDER,
+          }),
+        ],
+        format: format.combine(format.timestamp(), format.json()),
+      })
+  ) as winston.Logger;
+  wikiLoggers[workspaceID] = wikiLogger;
+  return wikiLogger;
+}
+
+export function getWikiLogger(workspaceID: string): winston.Logger {
+  return wikiLoggers[workspaceID];
+}
+
+export function stopWikiLogger(workspaceID: string) {
+  wikiLoggers[workspaceID].close();
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+    delete wikiLoggers[workspaceID];
+  } catch {}
 }

src/services/wiki/constants.ts

@@ -0,0 +1 @@
+export const wikiWorkerStartedEventName = (workspaceID: string) => `wikiWorkerStarted-${workspaceID}`;

src/services/wiki/index.ts

@@ -16,7 +16,7 @@ import type { IAuthenticationService } from '@services/auth/interface';
 import { lazyInject } from '@services/container';
 import type { IGitService, IGitUserInfos } from '@services/git/interface';
 import { i18n } from '@services/libs/i18n';
-import { getWikiLogFilePath, logger, refreshOutputFile, wikiOutputToFile } from '@services/libs/log';
+import { logger, startWikiLogger } from '@services/libs/log';
 import serviceIdentifier from '@services/serviceIdentifier';
 import { SupportedStorageServices } from '@services/types';
 import type { IViewService } from '@services/view/interface';
@@ -35,7 +35,9 @@ import { IDatabaseService } from '@services/database/interface';
 import { IPreferenceService } from '@services/preferences/interface';
 // @ts-expect-error it don't want .ts
 // eslint-disable-next-line import/no-webpack-loader-syntax
+import { mapValues } from 'lodash';
 import workerURL from 'threads-plugin/dist/loader?name=wikiWorker!./wikiWorker.ts';
+import { wikiWorkerStartedEventName } from './constants';
 import { IWikiOperations, wikiOperations } from './wikiOperations';
 
 @injectable()
@@ -93,6 +95,8 @@ export class Wiki implements IWikiService {
     return this.wikiWorkers[id];
   }
 
+  private readonly wikiWorkerStartedEventTarget = new EventTarget();
+
   public async startWiki(workspaceID: string, userName: string): Promise<void> {
     if (workspaceID === undefined) {
       logger.error('Try to start wiki, but workspace ID not provided', { workspaceID });
@@ -137,21 +141,19 @@ export class Wiki implements IWikiService {
       openDebugger: process.env.DEBUG_WORKER === 'true',
     };
     const worker = await spawn<WikiWorker>(new Worker(workerURL as string), { timeout: 1000 * 60 });
+    this.wikiWorkerStartedEventTarget.dispatchEvent(new Event(wikiWorkerStartedEventName(workspaceID)));
     this.wikiWorkers[id] = worker;
-    void refreshOutputFile(name);
+    const wikiLogger = startWikiLogger(workspaceID, name);
     const loggerMeta = { worker: 'NodeJSWiki', homePath: wikiFolderLocation };
     await new Promise<void>((resolve, reject) => {
       // handle native messages
       Thread.errors(worker).subscribe(async (error) => {
-        logger.error(error.message, { ...loggerMeta, ...error });
-        void wikiOutputToFile(name, error.message);
+        wikiLogger.error(error.message, { function: 'Thread.errors' });
         reject(new WikiRuntimeError(error, name, false));
       });
       Thread.events(worker).subscribe((event: WorkerEvent) => {
         if (event.type === 'message') {
-          const messageString = JSON.stringify(event.data);
-          void wikiOutputToFile(id, `${messageString}\n`);
-          logger.debug('wiki message', { ...event.data, ...loggerMeta });
+          wikiLogger.info('', { ...mapValues(event.data, (value: unknown) => typeof value === 'string' ? `${value.substring(0, 200)}... (substring(0, 200))` : String(value)) });
         } else if (event.type === 'termination') {
           delete this.wikiWorkers[id];
           const warningMessage = `NodeJSWiki ${id} Worker stopped (can be normal quit, or unexpected error, see other logs to determine)`;
@@ -167,7 +169,7 @@ export class Wiki implements IWikiService {
         packagePathBase: PACKAGE_PATH_BASE,
       }).subscribe(async (message) => {
         if (message.type === 'stderr' || message.type === 'stdout') {
-          void wikiOutputToFile(id, message.message);
+          logger.info(message.message, { function: 'initCacheDatabase' });
         }
       });
 
@@ -213,13 +215,19 @@ export class Wiki implements IWikiService {
             }
           }
         } else if (message.type === 'stderr' || message.type === 'stdout') {
-          void wikiOutputToFile(id, message.message);
+          wikiLogger.info(message.message, { function: 'startNodeJSWiki' });
         }
       });
     });
   }
 
   public async callWikiIpcServerRoute<NAME extends IpcServerRouteNames>(workspaceID: string, route: NAME, ...arguments_: Parameters<IpcServerRouteMethods[NAME]>) {
+    // wait for wiki worker started
+    await new Promise<void>(resolve => {
+      this.wikiWorkerStartedEventTarget.addEventListener(wikiWorkerStartedEventName(workspaceID), () => {
+        resolve();
+      });
+    });
     const worker = this.getWorker(workspaceID);
     if (worker === undefined) {
       logger.warning(`No wiki for ${workspaceID}. No running worker, means you call this function too early, or maybe tiddlywiki server in this workspace failed to start`, {

src/services/wiki/ipcServerRoutes.ts

@@ -198,8 +198,6 @@ export class IpcServerRoutes {
         renderType = renderType ?? this.wikiInstance.server.get('tiddler-render-type') ?? 'text/html';
         renderTemplate = renderTemplate ?? this.wikiInstance.server.get('tiddler-render-template') ?? '$:/core/templates/server/static.tiddler.html';
       }
-      // DEBUG: console renderType
-      console.log(`renderType`, renderType, 'renderTemplate', renderTemplate);
       const text = this.wikiInstance.wiki.renderTiddler(renderType, renderTemplate, { parseAsInline: true, variables: { currentTiddler: title } });
 
       // Naughty not to set a content-type, but it's the easiest way to ensure the browser will see HTML pages as HTML, and accept plain text tiddlers as CSS or JS