mirrors/TidGi-Desktop
1
Fork 0
mirror of https://github.com/tiddly-gittly/TidGi-Desktop.git synced 2025-12-15 15:10:31 -08:00
Code Issues Projects Releases Packages Wiki Activity

fix: properly cleanup unuseable llm instance

Browse source 
fixes #453
This commit is contained in:
linonetwo 2023-07-19 13:33:09 +08:00
parent 185eefd194
commit 3ca373a198
2 changed files with 38 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

41
src/services/languageModel/llmWorker/llamaCpp.ts
View file

@ -16,24 +16,29 @@ export async function loadLLama(
// use dynamic import cjs version to fix https://github.com/andywer/threads.js/issues/478
const { LLamaCpp } = await import('llama-node/dist/llm/llama-cpp.cjs');
subscriber?.next({ message: 'library loaded, new LLM now', ...loggerCommonMeta });
runnerInstance = new LLM(LLamaCpp);
const loadConfig: LoadConfig = {
enableLogging: true,
nCtx: 1024,
seed: 0,
f16Kv: false,
logitsAll: false,
vocabOnly: false,
useMlock: false,
embedding: false,
useMmap: true,
nGpuLayers: 0,
...loadConfigOverwrite,
};
subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
await runnerInstance.load(loadConfig);
subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
return runnerInstance;
try {
runnerInstance = new LLM(LLamaCpp);
const loadConfig: LoadConfig = {
enableLogging: true,
nCtx: 1024,
seed: 0,
f16Kv: false,
logitsAll: false,
vocabOnly: false,
useMlock: false,
embedding: false,
useMmap: true,
nGpuLayers: 0,
...loadConfigOverwrite,
};
subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
await runnerInstance.load(loadConfig);
subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
return runnerInstance;
} catch (error) {
unloadLLama();
throw error;
}
}
export function unloadLLama() {
runnerInstance = undefined;

25
src/services/languageModel/llmWorker/rwkvCpp.ts
View file

@ -16,16 +16,21 @@ export async function loadRwkv(
// use dynamic import cjs version to fix https://github.com/andywer/threads.js/issues/478
const { RwkvCpp } = await import('llama-node/dist/llm/rwkv-cpp.cjs');
subscriber?.next({ message: 'library loaded, new LLM now', ...loggerCommonMeta });
runnerInstance = new LLM(RwkvCpp);
const loadConfig: LoadConfig = {
enableLogging: true,
nThreads: 4,
...loadConfigOverwrite,
};
subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
await runnerInstance.load(loadConfig);
subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
return runnerInstance;
try {
runnerInstance = new LLM(RwkvCpp);
const loadConfig: LoadConfig = {
enableLogging: true,
nThreads: 4,
...loadConfigOverwrite,
};
subscriber?.next({ message: 'prepared to load instance', ...loggerCommonMeta, meta: { ...loggerCommonMeta.meta, loadConfigOverwrite } });
await runnerInstance.load(loadConfig);
subscriber?.next({ message: 'instance loaded', ...loggerCommonMeta });
return runnerInstance;
} catch (error) {
unloadRwkv();
throw error;
}
}
export function unloadRwkv() {
runnerInstance = undefined;