From 672c90a68cc8a605bae8da5de0797cbee67aabf2 Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 31 Oct 2026 18:33:51 -0500 Subject: [PATCH] core: metadata search index for large instances metadata/index.ts builds one index from the cached GET /metaData (apps, tables, processes, reports, widgets with label, name, app path, hidden flag) and searches it by exact, prefix, word prefix, substring and subsequence match, ranked then by label, paged with a total for more. generateMetaData makes a document of any size; the test indexes 1000 tables, 500 processes, 2000 widgets and 300 reports (0.57 MB: parse 3 ms, index 4 ms, slowest search 9 ms on this Mac). Co-Authored-By: Claude Fable 5.1 --- packages/core/src/index.ts | 1 + packages/core/src/metadata/index.ts | 188 ++++++++++++++++++++++ packages/core/src/test/meta-index.test.ts | 44 +++++ 3 files changed, 233 insertions(+) create mode 100644 packages/core/src/metadata/index.ts create mode 100644 packages/core/src/test/meta-index.test.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dbc8b49..5026bb9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -22,6 +22,7 @@ export * from './filter/complete.js' export * from './records/result-cursor.js' export * from './files/navigator.js' export * from './env/doctor.js' +export * from './metadata/index.js' export * from './query/quick-search.js' export * from './records/goto.js' export * from './records/copy.js' diff --git a/packages/core/src/metadata/index.ts b/packages/core/src/metadata/index.ts new file mode 100644 index 0000000..db6be0b --- /dev/null +++ b/packages/core/src/metadata/index.ts @@ -0,0 +1,188 @@ +/* + * Copyright 2026 QRun.IO, Inc. + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +/** + * A search index over the navigable metadata (apps, tables, processes, reports, widgets), built + * once from the cached GET /metaData document, so palettes, menus and trees on an instance with + * thousands of objects never walk the tree per keystroke. Matching: exact, prefix, word prefix, + * substring, then subsequence (fuzzy), over label and name; ranked by match then label. + */ + +import type { MetaDataDocument } from './cache.js' + +export type IndexKind = 'app' | 'table' | 'process' | 'report' | 'widget' + +export interface IndexEntry { + kind: IndexKind + name: string + label: string + /** Labels of the enclosing apps, outermost first (empty for objects outside the app tree). */ + path: string[] + /** Lower-cased label, name and path words, precomputed for matching. */ + readonly haystack: string + readonly labelLower: string + readonly nameLower: string + hidden: boolean + /** Whether it sits in the app tree (navigable) rather than only in the maps. */ + inTree: boolean +} + +export interface SearchOptions { + kinds?: IndexKind[] + limit?: number + offset?: number + includeHidden?: boolean +} + +export interface SearchResult { + entries: IndexEntry[] + /** How many matched in all (for "more…"). */ + total: number +} + +interface TreeNode { type?: string; name?: string; label?: string; children?: TreeNode[] } + +function entry(kind: IndexKind, name: string, label: string, path: string[], hidden: boolean, inTree: boolean): IndexEntry { + const labelLower = label.toLowerCase() + const nameLower = name.toLowerCase() + return { kind, name, label, path, hidden, inTree, labelLower, nameLower, haystack: `${labelLower} ${nameLower} ${path.join(' ').toLowerCase()}` } +} + +/** Subsequence match: every character of the needle appears in order. */ +function subsequence(needle: string, hay: string): boolean { + let i = 0 + for (let j = 0; j < hay.length && i < needle.length; j++) if (hay[j] === needle[i]) i++ + return i === needle.length +} + +/** A rank for one entry against a needle (lower is better; Infinity: no match). */ +function rank(e: IndexEntry, needle: string): number { + if (!needle) return 5 + if (e.labelLower === needle || e.nameLower === needle) return 0 + if (e.labelLower.startsWith(needle) || e.nameLower.startsWith(needle)) return 1 + if (e.labelLower.includes(` ${needle}`) || e.nameLower.includes(`.${needle}`) || e.nameLower.includes(`_${needle}`)) return 2 + if (e.labelLower.includes(needle) || e.nameLower.includes(needle)) return 3 + if (e.haystack.includes(needle)) return 4 + if (needle.length >= 2 && (subsequence(needle, e.labelLower) || subsequence(needle, e.nameLower))) return 6 + return Infinity +} + +export class MetaDataIndex { + readonly entries: IndexEntry[] = [] + readonly counts: Record = { app: 0, table: 0, process: 0, report: 0, widget: 0 } + /** Milliseconds the build took. */ + readonly buildMillis: number + private byKey = new Map() + + constructor(meta: MetaDataDocument) { + const started = performance.now() + const kinds: Record = { APP: 'app', TABLE: 'table', PROCESS: 'process', REPORT: 'report' } + const walk = (nodes: TreeNode[], path: string[]) => { + for (const n of nodes) { + const kind = kinds[String(n.type)] + if (kind && n.name) this.add(entry(kind, n.name, n.label ?? n.name, path, false, true)) + if (n.children?.length) walk(n.children, [...path, n.label ?? n.name ?? '']) + } + } + walk((meta.appTree ?? []) as TreeNode[], []) + const maps: Array<[IndexKind, Record | undefined]> = [ + ['app', meta.apps as never], ['table', meta.tables as never], ['process', meta.processes as never], ['report', meta.reports as never], ['widget', meta.widgets as never], + ] + for (const [kind, map] of maps) { + for (const [name, v] of Object.entries(map ?? {})) { + const existing = this.byKey.get(`${kind}:${name}`) + if (existing) { + existing.hidden = Boolean(v.isHidden) + continue + } + this.add(entry(kind, name, String(v.label ?? name), [], Boolean(v.isHidden), false)) + } + } + for (const e of this.entries) if (!e.hidden) this.counts[e.kind]++ + this.buildMillis = performance.now() - started + } + + private add(e: IndexEntry): void { + const key = `${e.kind}:${e.name}` + if (this.byKey.has(key)) return + this.byKey.set(key, e) + this.entries.push(e) + } + + get(kind: IndexKind, name: string): IndexEntry | undefined { + return this.byKey.get(`${kind}:${name}`) + } + + /** Matches by relevance then label; every word of the text must match. */ + search(text: string, options: SearchOptions = {}): SearchResult { + const words = text.toLowerCase().trim().split(/\s+/).filter(Boolean) + const kinds = options.kinds ? new Set(options.kinds) : undefined + const scored: Array<{ e: IndexEntry; r: number }> = [] + for (const e of this.entries) { + if (kinds && !kinds.has(e.kind)) continue + if (e.hidden && !options.includeHidden) continue + let r = 0 + for (const w of words) { + const wr = rank(e, w) + if (wr === Infinity) { + r = Infinity + break + } + r = Math.max(r, wr) + } + if (r !== Infinity) scored.push({ e, r }) + } + scored.sort((a, b) => a.r - b.r || Number(b.e.inTree) - Number(a.e.inTree) || a.e.label.localeCompare(b.e.label)) + const offset = options.offset ?? 0 + const limit = options.limit ?? 50 + return { entries: scored.slice(offset, offset + limit).map((s) => s.e), total: scored.length } + } + + /** "12 apps, 1,000 tables, 500 processes" for a header line. */ + describe(): string { + const parts: string[] = [] + for (const [kind, label] of [['app', 'apps'], ['table', 'tables'], ['process', 'processes'], ['report', 'reports'], ['widget', 'widgets']] as const) { + if (this.counts[kind]) parts.push(`${this.counts[kind].toLocaleString('en-US')} ${label}`) + } + return parts.join(', ') + } +} + +/** A generated metadata document of a given size (tests and the large-instance fake server). */ +export function generateMetaData(sizes: { apps?: number; tables: number; processes: number; widgets?: number; reports?: number }): MetaDataDocument { + const appCount = sizes.apps ?? 20 + const apps: Record = {} + const tables: Record = {} + const processes: Record = {} + const widgets: Record = {} + const reports: Record = {} + const words = ['order', 'invoice', 'customer', 'shipment', 'product', 'warehouse', 'vendor', 'payment', 'ledger', 'contact', 'carrier', 'price'] + const tree: TreeNode[] = [] + for (let a = 0; a < appCount; a++) { + const name = `app${a}` + const node: TreeNode = { type: 'APP', name, label: `App ${a} ${words[a % words.length]}`, children: [] } + tree.push(node) + apps[name] = { name, label: node.label, widgets: [] as string[], children: node.children } + } + for (let t = 0; t < sizes.tables; t++) { + const name = `${words[t % words.length]}_${t}` + const label = `${words[t % words.length]![0]!.toUpperCase()}${words[t % words.length]!.slice(1)} ${t}` + tables[name] = { name, label, insertPermission: true, readPermission: true } + tree[t % appCount]!.children!.push({ type: 'TABLE', name, label }) + } + for (let p = 0; p < sizes.processes; p++) { + const name = `process${p}` + const label = `Recalculate ${words[p % words.length]} ${p}` + processes[name] = { name, label } + tree[p % appCount]!.children!.push({ type: 'PROCESS', name, label }) + } + for (let w = 0; w < (sizes.widgets ?? 0); w++) { + const name = `widget${w}` + widgets[name] = { name, label: `Widget ${w}`, type: 'statistics', hasPermission: true } + ;(apps[`app${w % appCount}`] as { widgets: string[] }).widgets.push(name) + } + for (let r = 0; r < (sizes.reports ?? 0); r++) reports[`report${r}`] = { name: `report${r}`, label: `Report ${r}` } + return { apps: apps as never, appTree: tree, tables: tables as never, processes: processes as never, widgets: widgets as never, reports: reports as never, branding: { appName: 'Large Instance' } } +} diff --git a/packages/core/src/test/meta-index.test.ts b/packages/core/src/test/meta-index.test.ts new file mode 100644 index 0000000..3a3292f --- /dev/null +++ b/packages/core/src/test/meta-index.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2026 QRun.IO, Inc. + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { MetaDataIndex, generateMetaData } from '../metadata/index.js' + +test('a 1000-table, 500-process instance indexes and searches in milliseconds', () => { + const doc = generateMetaData({ tables: 1000, processes: 500, widgets: 2000, reports: 300 }) + const text = JSON.stringify(doc) + const parseStart = performance.now() + const parsed = JSON.parse(text) + const parseMs = performance.now() - parseStart + const index = new MetaDataIndex(parsed) + assert.equal(index.counts.table, 1000) + assert.equal(index.counts.process, 500) + assert.equal(index.counts.widget, 2000) + assert.ok(index.buildMillis < 200, `build ${index.buildMillis.toFixed(1)} ms`) + const times: number[] = [] + for (const q of ['inv', 'invoice 12', 'cust', 'recalc ship', 'shpmnt', 'x', 'order_99', 'app 3']) { + const t = performance.now() + const r = index.search(q, { limit: 50 }) + times.push(performance.now() - t) + assert.ok(r.entries.length <= 50) + } + assert.ok(Math.max(...times) < 50, `search max ${Math.max(...times).toFixed(1)} ms`) + console.log(` ${(text.length / 1e6).toFixed(2)} MB parse ${parseMs.toFixed(1)} ms, index ${index.buildMillis.toFixed(1)} ms, search max ${Math.max(...times).toFixed(2)} ms`) +}) + +test('ranking: exact, prefix, word prefix, substring, then fuzzy; every word must match', () => { + const index = new MetaDataIndex({ + appTree: [{ type: 'APP', name: 'sales', label: 'Sales', children: [{ type: 'TABLE', name: 'order', label: 'Order' }, { type: 'TABLE', name: 'orderLine', label: 'Order Line' }, { type: 'TABLE', name: 'backorder', label: 'Back Order' }] }], + tables: { order: { name: 'order', label: 'Order' }, secret: { name: 'secret', label: 'Secret orders', isHidden: true } }, + }) + assert.deepEqual(index.search('order').entries.map((e) => e.name), ['order', 'orderLine', 'backorder']) + assert.equal(index.search('ordr').total, 3, 'fuzzy: o-r-d-r in order') + assert.deepEqual(index.search('order line').entries.map((e) => e.name), ['orderLine']) + assert.equal(index.search('secret').total, 0) + assert.equal(index.search('secret', { includeHidden: true }).total, 1) + assert.deepEqual(index.get('table', 'order')!.path, ['Sales']) + assert.equal(index.search('', { kinds: ['app'] }).entries[0]!.name, 'sales') +})