diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dbc8b49..5026bb9 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -22,6 +22,7 @@ export * from './filter/complete.js' export * from './records/result-cursor.js' export * from './files/navigator.js' export * from './env/doctor.js' +export * from './metadata/index.js' export * from './query/quick-search.js' export * from './records/goto.js' export * from './records/copy.js' diff --git a/packages/core/src/metadata/index.ts b/packages/core/src/metadata/index.ts new file mode 100644 index 0000000..db6be0b --- /dev/null +++ b/packages/core/src/metadata/index.ts @@ -0,0 +1,188 @@ +/* + * Copyright 2026 QRun.IO, Inc. + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +/** + * A search index over the navigable metadata (apps, tables, processes, reports, widgets), built + * once from the cached GET /metaData document, so palettes, menus and trees on an instance with + * thousands of objects never walk the tree per keystroke. Matching: exact, prefix, word prefix, + * substring, then subsequence (fuzzy), over label and name; ranked by match then label. + */ + +import type { MetaDataDocument } from './cache.js' + +export type IndexKind = 'app' | 'table' | 'process' | 'report' | 'widget' + +export interface IndexEntry { + kind: IndexKind + name: string + label: string + /** Labels of the enclosing apps, outermost first (empty for objects outside the app tree). */ + path: string[] + /** Lower-cased label, name and path words, precomputed for matching. */ + readonly haystack: string + readonly labelLower: string + readonly nameLower: string + hidden: boolean + /** Whether it sits in the app tree (navigable) rather than only in the maps. */ + inTree: boolean +} + +export interface SearchOptions { + kinds?: IndexKind[] + limit?: number + offset?: number + includeHidden?: boolean +} + +export interface SearchResult { + entries: IndexEntry[] + /** How many matched in all (for "more…"). */ + total: number +} + +interface TreeNode { type?: string; name?: string; label?: string; children?: TreeNode[] } + +function entry(kind: IndexKind, name: string, label: string, path: string[], hidden: boolean, inTree: boolean): IndexEntry { + const labelLower = label.toLowerCase() + const nameLower = name.toLowerCase() + return { kind, name, label, path, hidden, inTree, labelLower, nameLower, haystack: `${labelLower} ${nameLower} ${path.join(' ').toLowerCase()}` } +} + +/** Subsequence match: every character of the needle appears in order. */ +function subsequence(needle: string, hay: string): boolean { + let i = 0 + for (let j = 0; j < hay.length && i < needle.length; j++) if (hay[j] === needle[i]) i++ + return i === needle.length +} + +/** A rank for one entry against a needle (lower is better; Infinity: no match). */ +function rank(e: IndexEntry, needle: string): number { + if (!needle) return 5 + if (e.labelLower === needle || e.nameLower === needle) return 0 + if (e.labelLower.startsWith(needle) || e.nameLower.startsWith(needle)) return 1 + if (e.labelLower.includes(` ${needle}`) || e.nameLower.includes(`.${needle}`) || e.nameLower.includes(`_${needle}`)) return 2 + if (e.labelLower.includes(needle) || e.nameLower.includes(needle)) return 3 + if (e.haystack.includes(needle)) return 4 + if (needle.length >= 2 && (subsequence(needle, e.labelLower) || subsequence(needle, e.nameLower))) return 6 + return Infinity +} + +export class MetaDataIndex { + readonly entries: IndexEntry[] = [] + readonly counts: Record = { app: 0, table: 0, process: 0, report: 0, widget: 0 } + /** Milliseconds the build took. */ + readonly buildMillis: number + private byKey = new Map() + + constructor(meta: MetaDataDocument) { + const started = performance.now() + const kinds: Record = { APP: 'app', TABLE: 'table', PROCESS: 'process', REPORT: 'report' } + const walk = (nodes: TreeNode[], path: string[]) => { + for (const n of nodes) { + const kind = kinds[String(n.type)] + if (kind && n.name) this.add(entry(kind, n.name, n.label ?? n.name, path, false, true)) + if (n.children?.length) walk(n.children, [...path, n.label ?? n.name ?? '']) + } + } + walk((meta.appTree ?? []) as TreeNode[], []) + const maps: Array<[IndexKind, Record | undefined]> = [ + ['app', meta.apps as never], ['table', meta.tables as never], ['process', meta.processes as never], ['report', meta.reports as never], ['widget', meta.widgets as never], + ] + for (const [kind, map] of maps) { + for (const [name, v] of Object.entries(map ?? {})) { + const existing = this.byKey.get(`${kind}:${name}`) + if (existing) { + existing.hidden = Boolean(v.isHidden) + continue + } + this.add(entry(kind, name, String(v.label ?? name), [], Boolean(v.isHidden), false)) + } + } + for (const e of this.entries) if (!e.hidden) this.counts[e.kind]++ + this.buildMillis = performance.now() - started + } + + private add(e: IndexEntry): void { + const key = `${e.kind}:${e.name}` + if (this.byKey.has(key)) return + this.byKey.set(key, e) + this.entries.push(e) + } + + get(kind: IndexKind, name: string): IndexEntry | undefined { + return this.byKey.get(`${kind}:${name}`) + } + + /** Matches by relevance then label; every word of the text must match. */ + search(text: string, options: SearchOptions = {}): SearchResult { + const words = text.toLowerCase().trim().split(/\s+/).filter(Boolean) + const kinds = options.kinds ? new Set(options.kinds) : undefined + const scored: Array<{ e: IndexEntry; r: number }> = [] + for (const e of this.entries) { + if (kinds && !kinds.has(e.kind)) continue + if (e.hidden && !options.includeHidden) continue + let r = 0 + for (const w of words) { + const wr = rank(e, w) + if (wr === Infinity) { + r = Infinity + break + } + r = Math.max(r, wr) + } + if (r !== Infinity) scored.push({ e, r }) + } + scored.sort((a, b) => a.r - b.r || Number(b.e.inTree) - Number(a.e.inTree) || a.e.label.localeCompare(b.e.label)) + const offset = options.offset ?? 0 + const limit = options.limit ?? 50 + return { entries: scored.slice(offset, offset + limit).map((s) => s.e), total: scored.length } + } + + /** "12 apps, 1,000 tables, 500 processes" for a header line. */ + describe(): string { + const parts: string[] = [] + for (const [kind, label] of [['app', 'apps'], ['table', 'tables'], ['process', 'processes'], ['report', 'reports'], ['widget', 'widgets']] as const) { + if (this.counts[kind]) parts.push(`${this.counts[kind].toLocaleString('en-US')} ${label}`) + } + return parts.join(', ') + } +} + +/** A generated metadata document of a given size (tests and the large-instance fake server). */ +export function generateMetaData(sizes: { apps?: number; tables: number; processes: number; widgets?: number; reports?: number }): MetaDataDocument { + const appCount = sizes.apps ?? 20 + const apps: Record = {} + const tables: Record = {} + const processes: Record = {} + const widgets: Record = {} + const reports: Record = {} + const words = ['order', 'invoice', 'customer', 'shipment', 'product', 'warehouse', 'vendor', 'payment', 'ledger', 'contact', 'carrier', 'price'] + const tree: TreeNode[] = [] + for (let a = 0; a < appCount; a++) { + const name = `app${a}` + const node: TreeNode = { type: 'APP', name, label: `App ${a} ${words[a % words.length]}`, children: [] } + tree.push(node) + apps[name] = { name, label: node.label, widgets: [] as string[], children: node.children } + } + for (let t = 0; t < sizes.tables; t++) { + const name = `${words[t % words.length]}_${t}` + const label = `${words[t % words.length]![0]!.toUpperCase()}${words[t % words.length]!.slice(1)} ${t}` + tables[name] = { name, label, insertPermission: true, readPermission: true } + tree[t % appCount]!.children!.push({ type: 'TABLE', name, label }) + } + for (let p = 0; p < sizes.processes; p++) { + const name = `process${p}` + const label = `Recalculate ${words[p % words.length]} ${p}` + processes[name] = { name, label } + tree[p % appCount]!.children!.push({ type: 'PROCESS', name, label }) + } + for (let w = 0; w < (sizes.widgets ?? 0); w++) { + const name = `widget${w}` + widgets[name] = { name, label: `Widget ${w}`, type: 'statistics', hasPermission: true } + ;(apps[`app${w % appCount}`] as { widgets: string[] }).widgets.push(name) + } + for (let r = 0; r < (sizes.reports ?? 0); r++) reports[`report${r}`] = { name: `report${r}`, label: `Report ${r}` } + return { apps: apps as never, appTree: tree, tables: tables as never, processes: processes as never, widgets: widgets as never, reports: reports as never, branding: { appName: 'Large Instance' } } +} diff --git a/packages/core/src/test/meta-index.test.ts b/packages/core/src/test/meta-index.test.ts new file mode 100644 index 0000000..3a3292f --- /dev/null +++ b/packages/core/src/test/meta-index.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright 2026 QRun.IO, Inc. + * Licensed under the Apache License, Version 2.0 (see LICENSE). + */ + +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { MetaDataIndex, generateMetaData } from '../metadata/index.js' + +test('a 1000-table, 500-process instance indexes and searches in milliseconds', () => { + const doc = generateMetaData({ tables: 1000, processes: 500, widgets: 2000, reports: 300 }) + const text = JSON.stringify(doc) + const parseStart = performance.now() + const parsed = JSON.parse(text) + const parseMs = performance.now() - parseStart + const index = new MetaDataIndex(parsed) + assert.equal(index.counts.table, 1000) + assert.equal(index.counts.process, 500) + assert.equal(index.counts.widget, 2000) + assert.ok(index.buildMillis < 200, `build ${index.buildMillis.toFixed(1)} ms`) + const times: number[] = [] + for (const q of ['inv', 'invoice 12', 'cust', 'recalc ship', 'shpmnt', 'x', 'order_99', 'app 3']) { + const t = performance.now() + const r = index.search(q, { limit: 50 }) + times.push(performance.now() - t) + assert.ok(r.entries.length <= 50) + } + assert.ok(Math.max(...times) < 50, `search max ${Math.max(...times).toFixed(1)} ms`) + console.log(` ${(text.length / 1e6).toFixed(2)} MB parse ${parseMs.toFixed(1)} ms, index ${index.buildMillis.toFixed(1)} ms, search max ${Math.max(...times).toFixed(2)} ms`) +}) + +test('ranking: exact, prefix, word prefix, substring, then fuzzy; every word must match', () => { + const index = new MetaDataIndex({ + appTree: [{ type: 'APP', name: 'sales', label: 'Sales', children: [{ type: 'TABLE', name: 'order', label: 'Order' }, { type: 'TABLE', name: 'orderLine', label: 'Order Line' }, { type: 'TABLE', name: 'backorder', label: 'Back Order' }] }], + tables: { order: { name: 'order', label: 'Order' }, secret: { name: 'secret', label: 'Secret orders', isHidden: true } }, + }) + assert.deepEqual(index.search('order').entries.map((e) => e.name), ['order', 'orderLine', 'backorder']) + assert.equal(index.search('ordr').total, 3, 'fuzzy: o-r-d-r in order') + assert.deepEqual(index.search('order line').entries.map((e) => e.name), ['orderLine']) + assert.equal(index.search('secret').total, 0) + assert.equal(index.search('secret', { includeHidden: true }).total, 1) + assert.deepEqual(index.get('table', 'order')!.path, ['Sales']) + assert.equal(index.search('', { kinds: ['app'] }).entries[0]!.name, 'sales') +})