Merge 12c46a86e2 into 4cd84a6ba1

core/modules/wiki.js

@@ -545,7 +545,7 @@ exports.getTiddlerBacklinks = function(targetTitle) {
 
 	if(!backlinks) {
 		backlinks = [];
-		this.forEachTiddler(function(title,tiddler) {
+		this.each(function(_tiddler,title) {
 			var links = self.getTiddlerLinks(title);
 			if(links.indexOf(targetTitle) !== -1) {
 				backlinks.push(title);

editions/test/tiddlers/tests/benchmarks/concept-summary-backlinks.md

@@ -0,0 +1,141 @@
+# TiddlyWiki Performance Optimization — getTiddlerBacklinks
+
+This document captures the context for the `getTiddlerBacklinks` optimization in `core/modules/wiki.js`.
+
+---
+
+## 1. The Optimization
+
+### Change: Replace `forEachTiddler()` with `each()` in the backlinks fallback path
+
+**Before:**
+```javascript
+exports.getTiddlerBacklinks = function(targetTitle) {
+    var self = this,
+        backIndexer = this.getIndexer("BackIndexer"),
+        backlinks = backIndexer && backIndexer.subIndexers.link.lookup(targetTitle);
+    if(!backlinks) {
+        backlinks = [];
+        this.forEachTiddler(function(title, tiddler) {
+            var links = self.getTiddlerLinks(title);
+            if(links.indexOf(targetTitle) !== -1) {
+                backlinks.push(title);
+            }
+        });
+        return backlinks;
+    }
+    return backlinks.slice(0);
+};
+```
+
+**After:**
+```javascript
+exports.getTiddlerBacklinks = function(targetTitle) {
+    var self = this,
+        backIndexer = this.getIndexer("BackIndexer"),
+        backlinks = backIndexer && backIndexer.subIndexers.link.lookup(targetTitle);
+    if(!backlinks) {
+        backlinks = [];
+        this.each(function(_tiddler, title) {
+            var links = self.getTiddlerLinks(title);
+            if(links.indexOf(targetTitle) !== -1) {
+                backlinks.push(title);
+            }
+        });
+        return backlinks;
+    }
+    return backlinks.slice(0);
+};
+```
+
+---
+
+## 2. Why `each()` is preferred over `forEachTiddler()`
+
+### Performance: `forEachTiddler()` sorts on every call
+
+`forEachTiddler()` (`core/modules/wiki.js`, line ~484) calls `this.getTiddlers(options)` internally, which:
+
+1. Collects all non-system tiddler titles
+2. **Sorts them alphabetically** via `sortTiddlers()` — O(n log n)
+3. Returns a new array
+
+This sort happens on **every call** to `getTiddlerBacklinks`. For a wiki with 10,000 tiddlers, that's an expensive sort each time — completely wasted work since backlinks scanning doesn't need any particular order.
+
+`each()` (`boot/boot.js`, line ~1284) simply iterates the internal tiddler hash directly via `getTiddlerTitles()`. No sorting, no filtering, no new array allocation.
+
+### Correctness: `forEachTiddler()` skips system tiddlers
+
+`forEachTiddler()` excludes system tiddlers (`$:/...` prefix) by default. This creates an inconsistency in `getTiddlerBacklinks`:
+
+- **BackIndexer path** (when available): `backIndexer.subIndexers.link.lookup()` indexes **all** tiddlers, including system tiddlers. If `$:/MyPlugin` links to `SomeTiddler`, the BackIndexer returns it as a backlink.
+- **Fallback path** (old code with `forEachTiddler`): Would **miss** backlinks from system tiddlers because they are filtered out.
+
+Using `each()` makes the fallback path consistent with the BackIndexer — both include system tiddlers. This fixes a subtle bug where the two code paths could return different results depending on whether the BackIndexer was available.
+
+### Summary
+
+| Aspect | `forEachTiddler()` | `each()` |
+|---|---|---|
+| Sorting | Sorts alphabetically every call — O(n log n) | No sort — direct iteration |
+| System tiddlers | Excluded by default | Included |
+| BackIndexer consistency | Inconsistent (misses `$:/` backlinks) | Consistent |
+| Callback signature | `function(title, tiddler)` | `function(tiddler, title)` |
+
+Note the **swapped callback parameter order**: `each()` passes `(tiddler, title)` while `forEachTiddler()` passes `(title, tiddler)`.
+
+---
+
+## 3. Why `extractLinks` was NOT optimized
+
+An earlier attempt replaced `indexOf` with `Object.create(null)` hash map in `extractLinks()` for O(1) deduplication. Benchmarks showed this was **slower** (~0.5x) for typical tiddlers because:
+
+- Real tiddlers have only 1-5 links — `indexOf` on a tiny array is faster than hash map overhead
+- TOC / table of contents pages use transclusions, not link nodes in the parse tree, so `extractLinks` never sees large arrays in practice
+- The pathological case (tiddlers with 50+ duplicate links) doesn't occur in real wikis
+
+The change was reverted — the optimization would never pay off in practice.
+
+---
+
+## 4. Benchmark Results
+
+| Metric | Old (`forEachTiddler`) | New (`each()`) | Speedup |
+|---|---|---|---|
+| Median (20 targets, 10k tiddlers) | ~112ms | ~19ms | **~6x faster** |
+
+---
+
+## 5. Benchmark Dual-Mode: Node Module + Browser Console
+
+`links-benchmark-core.js` works in three contexts:
+
+1. **Node test suite** — `require("links-benchmark-core.js")` returns `{ run: fn }`, called by the Jasmine wrapper
+2. **Standalone runner** — same `require()` path, called by `run-benchmark.js`
+3. **Browser console** — paste the entire file; it detects `typeof exports === "undefined"` and auto-runs with `$tw.wiki`
+
+`buildWiki($tw, wiki)` accepts an optional second argument:
+- **Omitted / falsy** — creates a fresh isolated `new $tw.Wiki({enableIndexers: []})`. Used by the Node test suite.
+- **Provided** (e.g., `$tw.wiki`) — adds tiddlers to the existing live wiki. Used when pasted into the browser console.
+
+Both modes produce **identical tiddlers** — same titles (e.g., `"Tiddler0"`), same content, same seeded PRNG, same percentages. No prefixes, no extra fields (tags, etc.). This ensures benchmark results are comparable across environments.
+
+In the browser, tiddlers persist after the benchmark — they are **not** cleaned up. Find them via `[prefix[Tiddler]]` or `[prefix[MissingTiddler]]` in Advanced Search.
+
+---
+
+## 6. Design Rules
+
+1. **Keep test tiddlers identical across environments** — Do not add tags, prefixes, extra fields, or any data that the isolated wiki mode doesn't add. Any difference changes test conditions (e.g., tags affect link parsing, prefixes change titles), making results non-comparable. Both modes must produce the exact same tiddlers.
+
+---
+
+## 7. File Locations
+
+- **Optimized source:** `core/modules/wiki.js` — `getTiddlerBacklinks` (line ~543)
+- **`each()` definition:** `boot/boot.js` (line ~1284)
+- **`forEachTiddler()` definition:** `core/modules/wiki.js` (line ~484)
+- **BackIndexer:** `core/modules/indexers/back-indexer.js` (line ~9)
+- **Benchmark core:** `editions/test/tiddlers/tests/benchmarks/links-benchmark-core.js`
+- **Jasmine wrapper:** `editions/test/tiddlers/tests/benchmarks/test-links-benchmark.js`
+- **Standalone runner:** `editions/test/tiddlers/tests/benchmarks/run-benchmark.js`

editions/test/tiddlers/tests/benchmarks/concept-summary-backlinks.md.meta

@@ -0,0 +1,4 @@
+title: Backlinks Titles Benchmark Concept
+modified: 20260308233435000
+created: 20260308233435000
+type: text/plain

editions/test/tiddlers/tests/benchmarks/links-benchmark-core.js

@@ -0,0 +1,238 @@
+/*\
+title: links-benchmark-core.js
+type: application/javascript
+module-type: library
+
+Shared benchmark code for getTiddlerBacklinks optimization.
+Used by both the Jasmine test (test-links-benchmark.js) and
+the standalone runner (run-benchmark.js).
+
+Usage:
+  var benchmark = require("links-benchmark-core.js");
+  var results = benchmark.run($tw);
+
+\*/
+"use strict";
+
+var now = (typeof performance !== "undefined" && typeof performance.now === "function")
+	? performance.now.bind(performance)
+	: function() {
+		var hr = process.hrtime();
+		return hr[0] * 1000 + hr[1] / 1e6;
+	};
+
+var TIDDLER_COUNT = 10000;
+var LINK_PERCENTAGE = 0.10;       // 10% of tiddlers link to other tiddlers
+var NO_LINK_PERCENTAGE = 0.20;    // 20% of tiddlers have no links at all
+var MISSING_LINK_PERCENTAGE = 0.10; // 10% of link targets are non-existent tiddlers
+var LINKS_PER_TIDDLER_MIN = 1;
+var LINKS_PER_TIDDLER_MAX = 5;
+var WARMUP_RUNS = 2;
+var BENCHMARK_RUNS = 5;
+// Run multiple iterations per timed sample to overcome low-resolution browser timers
+var ITERATIONS_PER_SAMPLE = 10;
+
+// Seeded PRNG for reproducible benchmarks
+function mulberry32(seed) {
+	return function() {
+		seed |= 0; seed = seed + 0x6D2B79F5 | 0;
+		var t = Math.imul(seed ^ seed >>> 15, 1 | seed);
+		t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
+		return ((t ^ t >>> 14) >>> 0) / 4294967296;
+	};
+}
+
+// Old getTiddlerBacklinks: uses forEachTiddler (sorts + filters system tiddlers)
+function getTiddlerBacklinksOld(wiki, targetTitle) {
+	var backlinks = [];
+	wiki.forEachTiddler(function(title, tiddler) {
+		var links = wiki.getTiddlerLinks(title);
+		if(links.indexOf(targetTitle) !== -1) {
+			backlinks.push(title);
+		}
+	});
+	return backlinks;
+}
+
+// New getTiddlerBacklinks: uses each() (no sort, includes all tiddlers)
+function getTiddlerBacklinksNew(wiki, targetTitle) {
+	var backlinks = [];
+	wiki.each(function(_tiddler, title) {
+		var links = wiki.getTiddlerLinks(title);
+		if(links.indexOf(targetTitle) !== -1) {
+			backlinks.push(title);
+		}
+	});
+	return backlinks;
+}
+
+/*
+Build test tiddlers and add them to a wiki.
+  $tw   - the TiddlyWiki instance (must be booted)
+  wiki  - (optional) wiki to add tiddlers to. If omitted a fresh
+          isolated wiki is created (used by the Node test suite).
+          Identical tiddlers are produced in both modes.
+*/
+function buildWiki($tw, wiki) {
+	var random = mulberry32(42);
+	if(!wiki) {
+		wiki = new $tw.Wiki({enableIndexers: []});
+		wiki.addIndexersToWiki();
+	}
+	var allTitles = [];
+	var missingTitles = [];
+	var linkingTiddlers = [];
+	var t;
+	for(t = 0; t < TIDDLER_COUNT; t++) {
+		allTitles.push("Tiddler" + t);
+	}
+	var missingCount = Math.floor(TIDDLER_COUNT * MISSING_LINK_PERCENTAGE);
+	for(t = 0; t < missingCount; t++) {
+		missingTitles.push("MissingTiddler" + t);
+	}
+	var allTargets = allTitles.concat(missingTitles);
+	var noLinkCount = Math.floor(TIDDLER_COUNT * NO_LINK_PERCENTAGE);
+	var linkingCount = Math.floor(TIDDLER_COUNT * LINK_PERCENTAGE);
+	var indices = [];
+	for(t = 0; t < TIDDLER_COUNT; t++) {
+		indices.push(t);
+	}
+	for(t = indices.length - 1; t > 0; t--) {
+		var j = Math.floor(random() * (t + 1));
+		var temp = indices[t];
+		indices[t] = indices[j];
+		indices[j] = temp;
+	}
+	var noLinkSet = Object.create(null);
+	for(t = 0; t < noLinkCount; t++) {
+		noLinkSet[indices[t]] = true;
+	}
+	var linkingSet = Object.create(null);
+	for(t = noLinkCount; t < noLinkCount + linkingCount; t++) {
+		linkingSet[indices[t]] = true;
+	}
+	for(t = 0; t < TIDDLER_COUNT; t++) {
+		var text;
+		if(noLinkSet[t]) {
+			text = "This is tiddler " + t + " with no links.";
+		} else if(linkingSet[t]) {
+			var numLinks = LINKS_PER_TIDDLER_MIN + Math.floor(random() * (LINKS_PER_TIDDLER_MAX - LINKS_PER_TIDDLER_MIN + 1));
+			var links = [];
+			for(var l = 0; l < numLinks; l++) {
+				var targetIdx = Math.floor(random() * allTargets.length);
+				links.push("[[" + allTargets[targetIdx] + "]]");
+			}
+			text = "Tiddler " + t + " links to " + links.join(" and ");
+			linkingTiddlers.push(allTitles[t]);
+		} else {
+			text = "Content of tiddler " + t + ".";
+		}
+		wiki.addTiddler({
+			title: allTitles[t],
+			text: text
+		});
+	}
+	return { wiki: wiki, allTitles: allTitles, missingTitles: missingTitles, linkingTiddlers: linkingTiddlers };
+}
+
+function benchmarkFn(fn, label) {
+	var r, i;
+	for(r = 0; r < WARMUP_RUNS; r++) {
+		fn();
+	}
+	var times = [];
+	var result;
+	for(r = 0; r < BENCHMARK_RUNS; r++) {
+		var start = now();
+		for(i = 0; i < ITERATIONS_PER_SAMPLE; i++) {
+			result = fn();
+		}
+		var end = now();
+		times.push((end - start) / ITERATIONS_PER_SAMPLE);
+	}
+	times.sort(function(a, b) { return a - b; });
+	var median = times[Math.floor(times.length / 2)];
+	var avg = times.reduce(function(s, v) { return s + v; }, 0) / times.length;
+	var min = times[0];
+	var max = times[times.length - 1];
+	console.log("  " + label + ": median=" + median.toFixed(2) + "ms, avg=" + avg.toFixed(2) + "ms, min=" + min.toFixed(2) + "ms, max=" + max.toFixed(2) + "ms");
+	return { result: result, median: median, avg: avg, min: min, max: max };
+}
+
+/*
+Run all benchmarks. Returns an object with results for use by callers.
+  $tw   - the TiddlyWiki instance (must be booted)
+  wiki  - (optional) existing wiki to add tiddlers to
+*/
+function run($tw, wiki) {
+	console.log("\nBuilding wiki with " + TIDDLER_COUNT + " tiddlers...");
+	var buildStart = now();
+	var data = buildWiki($tw, wiki);
+	var benchWiki = data.wiki;
+	var buildElapsed = now() - buildStart;
+	console.log("Wiki built in " + buildElapsed.toFixed(0) + "ms");
+	console.log("  " + TIDDLER_COUNT + " tiddlers, " +
+		Math.floor(TIDDLER_COUNT * LINK_PERCENTAGE) + " linking, " +
+		Math.floor(TIDDLER_COUNT * NO_LINK_PERCENTAGE) + " with no links, " +
+		data.missingTitles.length + " missing targets");
+
+	// Pick a subset of target titles that have backlinks for meaningful testing
+	var backlinkTargets = [];
+	for(var b = 0; b < data.linkingTiddlers.length && backlinkTargets.length < 20; b++) {
+		var links = benchWiki.getTiddlerLinks(data.linkingTiddlers[b]);
+		for(var lb = 0; lb < links.length && backlinkTargets.length < 20; lb++) {
+			if(benchWiki.tiddlerExists(links[lb])) {
+				backlinkTargets.push(links[lb]);
+			}
+		}
+	}
+	console.log("  " + backlinkTargets.length + " target titles for backlinks benchmark");
+
+	// getTiddlerBacklinks correctness
+	var backlinksCorrect = true;
+	for(var bc = 0; bc < backlinkTargets.length; bc++) {
+		var oldBacklinks = getTiddlerBacklinksOld(benchWiki, backlinkTargets[bc]).slice().sort();
+		var newBacklinks = getTiddlerBacklinksNew(benchWiki, backlinkTargets[bc]).slice().sort();
+		if(JSON.stringify(oldBacklinks) !== JSON.stringify(newBacklinks)) {
+			// The new version uses each() which includes system tiddlers,
+			// while the old uses forEachTiddler which excludes them.
+			// In our test wiki there are no system tiddlers, so results should match.
+			backlinksCorrect = false;
+			break;
+		}
+	}
+
+	// getTiddlerBacklinks performance
+	console.log("\n  getTiddlerBacklinks benchmark (" + BENCHMARK_RUNS + " runs, " + WARMUP_RUNS + " warmup, " + ITERATIONS_PER_SAMPLE + " iter/sample):");
+	var backlinksOldBench = benchmarkFn(function() {
+		var results = [];
+		for(var i = 0; i < backlinkTargets.length; i++) {
+			results.push(getTiddlerBacklinksOld(benchWiki, backlinkTargets[i]));
+		}
+		return results;
+	}, "OLD (forEachTiddler)  ");
+	var backlinksNewBench = benchmarkFn(function() {
+		var results = [];
+		for(var i = 0; i < backlinkTargets.length; i++) {
+			results.push(getTiddlerBacklinksNew(benchWiki, backlinkTargets[i]));
+		}
+		return results;
+	}, "NEW (each)            ");
+	var backlinksSpeedup = backlinksOldBench.median / backlinksNewBench.median;
+	console.log("  Speedup: " + backlinksSpeedup.toFixed(2) + "x faster");
+
+	return {
+		correct: backlinksCorrect,
+		targetCount: backlinkTargets.length,
+		oldMedian: backlinksOldBench.median,
+		newMedian: backlinksNewBench.median,
+		speedup: backlinksSpeedup
+	};
+}
+
+// Export for Node/TiddlyWiki module system, auto-run for browser console
+if(typeof exports !== "undefined") {
+	exports.run = run;
+} else {
+	run($tw, $tw.wiki);
+}

editions/test/tiddlers/tests/benchmarks/run-benchmark.js

@@ -0,0 +1,90 @@
+/*
+Standalone benchmark runner for TiddlyWiki performance tests.
+Boots TW core minimally and runs benchmarks directly — much faster
+than the full Jasmine test suite on Windows.
+
+Automatically discovers and runs all *-benchmark-core.js files in
+the same directory. Missing core files (from other branches) are
+skipped gracefully.
+
+Usage:
+  node editions/test/tiddlers/tests/benchmarks/run-benchmark.js
+*/
+
+"use strict";
+
+var path = require("path");
+var fs = require("fs");
+
+// Boot TiddlyWiki with just the core (no editions, no plugins, no Jasmine)
+var $tw = require("../../../../../boot/boot.js").TiddlyWiki();
+$tw.boot.argv = [];
+// Suppress boot help/info output, restore before running benchmarks
+var _write = process.stdout.write;
+process.stdout.write = function() { return true; };
+$tw.boot.boot(function() {
+	process.stdout.write = _write;
+
+	console.log("TiddlyWiki " + $tw.version + " — Standalone Benchmark Runner\n");
+
+	// Discover all *-benchmark-core.js files in this directory
+	var benchmarkDir = __dirname;
+	var files = fs.readdirSync(benchmarkDir);
+	var coreFiles = files.filter(function(f) {
+		return f.match(/-benchmark-core\.js$/);
+	}).sort();
+
+	if(coreFiles.length === 0) {
+		console.log("No benchmark core files found in " + benchmarkDir);
+		process.exit(0);
+	}
+
+	var allPassed = true;
+	var ranCount = 0;
+
+	coreFiles.forEach(function(coreFile) {
+		var fullPath = path.join(benchmarkDir, coreFile);
+		console.log("\n" + "=".repeat(60));
+		console.log("Running: " + coreFile);
+		console.log("=".repeat(60));
+
+		try {
+			var benchmark = require(fullPath);
+			var results = benchmark.run($tw);
+			ranCount++;
+
+			// Check correctness — handle both flat and nested result formats
+			var correct = checkCorrectness(results);
+			console.log("\nCorrectness: " + (correct ? "PASS" : "FAIL"));
+			if(!correct) {
+				console.error("ERROR: Old and new implementations return different results!");
+				allPassed = false;
+			}
+		} catch(e) {
+			console.error("ERROR running " + coreFile + ": " + e.message);
+			allPassed = false;
+		}
+	});
+
+	console.log("\n" + "=".repeat(60));
+	console.log("Ran " + ranCount + "/" + coreFiles.length + " benchmark(s)");
+	console.log("=".repeat(60));
+
+	process.exit(allPassed ? 0 : 1);
+});
+
+// Check correctness for both flat results (e.g. {correct: true})
+// and nested results (e.g. {extractLinks: {correct: true}, backlinks: {correct: true}})
+function checkCorrectness(results) {
+	if(typeof results.correct === "boolean") {
+		return results.correct;
+	}
+	var keys = Object.keys(results);
+	for(var i = 0; i < keys.length; i++) {
+		var sub = results[keys[i]];
+		if(sub && typeof sub.correct === "boolean" && !sub.correct) {
+			return false;
+		}
+	}
+	return true;
+}

editions/test/tiddlers/tests/benchmarks/run-benchmark.js.meta

@@ -0,0 +1,5 @@
+title: Run Benchmark on Windows - small wrapper - much faster
+type: text/plain
+created: 20260308204959
+modified: 20260308205023000
+

editions/test/tiddlers/tests/benchmarks/test-links-benchmark.js

@@ -0,0 +1,31 @@
+/*\
+title: test-links-benchmark.js
+type: application/javascript
+tags: [[$:/tags/test-spec]]
+
+Performance benchmark for getTiddlerBacklinks optimization.
+Delegates to links-benchmark-core.js for the actual benchmark logic.
+
+\*/
+"use strict";
+
+// TODO: Adjust the version check for the target release
+if($tw.version.indexOf("5.4.0") === 0) {
+
+	var benchmark = require("links-benchmark-core.js");
+
+	describe("Backlink performance benchmarks", function() {
+
+		var results = benchmark.run($tw);
+
+		it("correctness: getTiddlerBacklinks new implementation should return the same results as old", function() {
+			expect(results.correct).toBe(true);
+			console.log("  getTiddlerBacklinks: " + results.targetCount + " target titles tested");
+		});
+
+		it("performance: getTiddlerBacklinks new implementation should be faster than old", function() {
+			expect(results.newMedian).toBeLessThan(results.oldMedian);
+			console.log("  Speedup: " + results.speedup.toFixed(2) + "x faster");
+		});
+	});
+}