From 285ef84b7ed0d99535f6c431c504ff41b737d756 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Mon, 4 May 2026 16:58:48 -0400 Subject: [PATCH] Refine journal search filtering --- .../BOOTSTRAP__AI_Agent_Quickstart.md | 5 + .../GUIDE__SvelteKit2_Svelte5_DexieJS.md | 2 + .../ae_journals/ae_journals_search_helpers.ts | 105 +++++++++ .../journals/[journal_id]/+layout.svelte | 2 +- src/routes/journals/[journal_id]/+page.svelte | 199 ++++++++++-------- .../ae_comp__journal_entry_obj_li.svelte | 25 +-- .../ae_comp__journal_entry_obj_qry.svelte | 103 +++++---- .../journals/journal_entry_visibility.test.ts | 162 ++++++++------ vitest.config.ts | 18 +- 9 files changed, 379 insertions(+), 242 deletions(-) create mode 100644 src/lib/ae_journals/ae_journals_search_helpers.ts diff --git a/documentation/BOOTSTRAP__AI_Agent_Quickstart.md b/documentation/BOOTSTRAP__AI_Agent_Quickstart.md index 61a2a21b..0757fbee 100644 --- a/documentation/BOOTSTRAP__AI_Agent_Quickstart.md +++ b/documentation/BOOTSTRAP__AI_Agent_Quickstart.md @@ -306,6 +306,11 @@ These are real incidents — know them before you start. redundant on the V3 path. Both paths now pretty-print with 2-space indent. See `GUIDE__AE_API_V3_for_Frontend.md` → section 3C for the full explanation. +12. **Broad Dexie result windows get silently clipped** — if a broad "All" view shows fewer + rows than a narrower filter, check for a page-level limit or an API revalidation step + replacing the local IDB result set. For empty text searches, the full local result set + should drive the display; server refreshes should update cache, not shrink visibility. + --- ## 8. Source Layout (Quick Reference) diff --git a/documentation/GUIDE__SvelteKit2_Svelte5_DexieJS.md b/documentation/GUIDE__SvelteKit2_Svelte5_DexieJS.md index c811627c..a81feb82 100644 --- a/documentation/GUIDE__SvelteKit2_Svelte5_DexieJS.md +++ b/documentation/GUIDE__SvelteKit2_Svelte5_DexieJS.md @@ -91,6 +91,7 @@ $effect(() => { ## Practical Patterns from Aether (Journals & Events) - Journals: The journaling pages use SWR-style background refreshes but reliably render because either (a) the page `+page.ts` blocks to populate DB for critical views, or (b) components accept `data.initial_*` fallback values until `liveQuery` emits. This hybrid approach avoids the "refresh twice" problem while keeping navigation snappy. +- Journals broad views: if text search is empty, let the local IDB result set drive the visible list. The API can revalidate the cache in the background, but it should not replace a broad "All" view with a limited slice that hides valid rows. - Sessions / Presentations: The session page demonstrates several best practices: - Use `url_*` constants (derived from `data.params`) so the `liveQuery` closure captures a stable value instead of the reactive store directly. @@ -113,6 +114,7 @@ let lq__event_presentation_obj_li = $derived( - Add a small `console.log` inside each `liveQuery` closure to confirm when it runs and what `id` it sees. - Verify that `+page.ts` either `await`s critical loads or returns `initial_*` payloads for first-render hydration. - Confirm that dependent store values (selected IDs) are assigned before components subscribe — use `untrack` to prevent extra reactive cycles. +- If a broad Dexie-backed list shows fewer rows than a narrower filter, look for a limit or revalidation step overwriting the local IDB result set. Broad views should stay unbounded unless the user is actually narrowing by text. - Ensure your `liveQuery` closures return quickly and do not throw; any exception inside the query can stop updates. - If a dependent query appears stale, temporarily add `await 0` in the upstream query or an explicit `Promise.resolve()` after the IDB write to force the microtask queue to flush during debugging. diff --git a/src/lib/ae_journals/ae_journals_search_helpers.ts b/src/lib/ae_journals/ae_journals_search_helpers.ts new file mode 100644 index 00000000..bfe6de4f --- /dev/null +++ b/src/lib/ae_journals/ae_journals_search_helpers.ts @@ -0,0 +1,105 @@ +export interface JournalEntrySearchParams { + str?: string; + cat?: string | null; + enabled?: 'enabled' | 'all' | 'not_enabled'; + hidden?: 'hidden' | 'all' | 'not_hidden'; + limit?: number; +} + +function normalize_search_value(value: unknown): string { + if (value === null || value === undefined) return ''; + + if (typeof value === 'string') return value.toLowerCase(); + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value).toLowerCase(); + } + + try { + return JSON.stringify(value).toLowerCase(); + } catch { + return String(value).toLowerCase(); + } +} + +function build_journal_entry_search_blob(entry: any): string { + return [ + entry?.code, + entry?.name, + entry?.short_name, + entry?.summary, + entry?.outline, + entry?.content, + entry?.history, + entry?.notes, + entry?.tags, + entry?.activity_code, + entry?.category_code, + entry?.type_code, + entry?.topic_code, + entry?.group, + entry?.journal_code, + entry?.journal_name, + entry?.alert_msg, + entry?.default_qry_str + ] + .map(normalize_search_value) + .filter(Boolean) + .join(' '); +} + +export function journal_entry_matches_search( + entry: any, + params: JournalEntrySearchParams +): boolean { + if (!entry) return false; + + const qry_str = (params.str ?? '').toLowerCase().trim(); + const category_code = params.cat ?? ''; + const enabled_mode = params.enabled ?? 'all'; + const hidden_mode = params.hidden ?? 'all'; + + const is_hidden = entry.hide === true || entry.hide === 1; + const is_disabled = entry.enable === false || entry.enable === 0; + + if (category_code && entry.category_code !== category_code) return false; + + if (enabled_mode === 'enabled' && is_disabled) return false; + if (enabled_mode === 'not_enabled' && !is_disabled) return false; + + if (hidden_mode === 'hidden' && !is_hidden) return false; + if (hidden_mode === 'not_hidden' && is_hidden) return false; + + if (!qry_str) return true; + + return build_journal_entry_search_blob(entry).includes(qry_str); +} + +export function journal_entry_compare_for_list(a: any, b: any): number { + return ( + (b?.tmp_sort_1 ?? '').localeCompare(a?.tmp_sort_1 ?? '') || + (b?.updated_on ?? '').localeCompare(a?.updated_on ?? '') || + (b?.journal_entry_id ?? '').localeCompare(a?.journal_entry_id ?? '') + ); +} + +export function journal_entry_filter_list( + list: any[] | null | undefined, + params: JournalEntrySearchParams +): any[] | null { + if (list === undefined || list === null) return null; + if (!Array.isArray(list)) return []; + + const has_text_search = Boolean((params.str ?? '').trim()); + + const filtered = list + .filter((entry) => journal_entry_matches_search(entry, params)) + .sort(journal_entry_compare_for_list); + + // Broad views should show the full local result set; only text searches + // should be sliced to a page-sized window. + if (has_text_search && params.limit && params.limit > 0) { + return filtered.slice(0, params.limit); + } + + return filtered; +} \ No newline at end of file diff --git a/src/routes/journals/[journal_id]/+layout.svelte b/src/routes/journals/[journal_id]/+layout.svelte index 8ce3fef0..97e998f8 100644 --- a/src/routes/journals/[journal_id]/+layout.svelte +++ b/src/routes/journals/[journal_id]/+layout.svelte @@ -235,7 +235,7 @@ Middle-click to open in new tab`}> {:else} - + {/if} diff --git a/src/routes/journals/[journal_id]/+page.svelte b/src/routes/journals/[journal_id]/+page.svelte index 569a4602..228cadd3 100644 --- a/src/routes/journals/[journal_id]/+page.svelte +++ b/src/routes/journals/[journal_id]/+page.svelte @@ -3,7 +3,10 @@ let log_lvl: number = $state(0); interface Props { - data: any; + data: { + account_id: string; + [key: string]: unknown; + }; } let { data }: Props = $props(); @@ -16,23 +19,13 @@ import { untrack } from 'svelte'; import { liveQuery } from 'dexie'; // *** Import Aether specific variables and functions -import { ae_util } from '$lib/ae_utils/ae_utils'; -import { - ae_snip, - ae_loc, - ae_sess, - ae_api, - ae_trig, - slct, - slct_trigger -} from '$lib/stores/ae_stores'; +import { ae_loc, ae_api } from '$lib/stores/ae_stores'; import { db_journals } from '$lib/ae_journals/db_journals'; +import { journal_entry_filter_list } from '$lib/ae_journals/ae_journals_search_helpers'; import { journals_loc, journals_sess, - journals_slct, - journals_prom, - journals_trig + journals_slct } from '$lib/ae_journals/ae_journals_stores'; import { journals_func } from '$lib/ae_journals/ae_journals_functions'; @@ -42,12 +35,34 @@ import AeCompModalJournalExport from '../ae_comp__modal_journal_export.svelte'; import AeCompModalJournalImport from '../ae_comp__modal_journal_import.svelte'; // Variables -let ae_acct = $derived(data[data.account_id]); +interface JournalPageAccount { + api?: unknown; + loc?: { + person_id?: string | null; + }; + slct: { + journal_id: string | null; + journal_entry_id?: string | null; + }; +} + +let ae_acct = $derived(data[data.account_id] as JournalPageAccount); let show_export_modal = $state(false); let show_import_modal = $state(false); +interface JournalSearchParams { + v: number; + str: string; + cat: string; + limit: number; + enabled: 'enabled' | 'all' | 'not_enabled'; + hidden: 'hidden' | 'all' | 'not_hidden'; + journal_id: string | null; + remote_first: boolean; +} + let search_id_li: Array = $state([]); -let search_debounce_timer: any = null; +let search_debounce_timer: ReturnType | null = null; let last_search_id = 0; let last_executed_key = ''; // Search Guard Key @@ -72,45 +87,6 @@ let lq__journal_obj = $derived( }) ); -// Stable LiveQuery Pattern (Aether UI V3) -// Re-wrapped in $derived to ensure the observable instance remains stable -// unless the underlying dependencies (ids, search context) change. -// Important: keep the `liveQuery` closure free of transient reactive -// references — capture stable values (ids, search keys) so the observable -// isn't recreated unnecessarily on every render. Use `search_id_li` or -// other plain arrays/values as explicit dependencies. -let lq__journal_entry_obj_li = $derived( - liveQuery(async () => { - const ids = search_id_li; - const journal_id = $lq__journal_obj?.journal_id; - const search_text = $journals_loc.entry.qry__search_text; - const cat_code = $journals_loc.entry.qry__category_code; - - // SCENARIO 1: Specific IDs provided (Search Results) - if (Array.isArray(ids) && ids.length > 0) { - if (log_lvl) - console.log(`Journal Page LQ: bulkGet ${ids.length} IDs`); - const results = await db_journals.journal_entry.bulkGet(ids); - return results.filter((item) => item !== undefined); - } - - // SCENARIO 2: Fallback to broad search (Default view) - if (journal_id && !search_text && !cat_code) { - if (log_lvl) - console.log( - `Journal Page LQ: Fallback search for journal: ${journal_id}` - ); - return await db_journals.journal_entry - .where('journal_id') - .equals(journal_id) - .reverse() - .sortBy('tmp_sort_1'); - } - - return []; - }) -); - // Standardized Reactive Search Pattern (Aether UI V3) // 1. Isolate dependencies into a stable derived object let search_params = $derived({ @@ -121,10 +97,53 @@ let search_params = $derived({ enabled: $journals_loc.entry.qry__enabled, hidden: $journals_loc.entry.qry__hidden, journal_id: $journals_slct.journal_id, - person_id: $ae_loc.person_id, remote_first: $journals_loc.entry.qry__remote_first }); +// Stable LiveQuery Pattern (Aether UI V3) +// Re-wrapped in $derived to ensure the observable instance remains stable +// unless the underlying dependencies (ids, search context) change. +// Important: keep the `liveQuery` closure free of transient reactive +// references — capture stable values (ids, search keys) so the observable +// isn't recreated unnecessarily on every render. Use `search_id_li` or +// other plain arrays/values as explicit dependencies. +let lq__journal_entry_obj_li = $derived( + (() => { + const ids = search_id_li; + const params = search_params; + const journal_id = $lq__journal_obj?.journal_id; + + return liveQuery(async () => { + if (params.remote_first && (!Array.isArray(ids) || ids.length === 0)) { + return null; + } + + // SCENARIO 1: Specific IDs provided (Search Results) + if (Array.isArray(ids) && ids.length > 0) { + if (log_lvl) + console.log(`Journal Page LQ: bulkGet ${ids.length} IDs`); + const results = await db_journals.journal_entry.bulkGet(ids); + return results.filter((item) => item !== undefined); + } + + if (!journal_id) return null; + + // SCENARIO 2: Fallback to broad journal search (Default view) + if (log_lvl) + console.log( + `Journal Page LQ: Broad search for journal: ${journal_id}` + ); + + const results = await db_journals.journal_entry + .where('journal_id') + .equals(journal_id) + .toArray(); + + return journal_entry_filter_list(results, params) ?? []; + }); + })() +); + // 2. Controlled effect for triggering searches $effect(() => { // Establishes reactive dependency on search_params @@ -143,7 +162,7 @@ $effect(() => { }; }); -async function handle_search_refresh(params: any) { +async function handle_search_refresh(params: JournalSearchParams) { // 1. Guard: Check if criteria actually changed const qry_key = JSON.stringify(params); if (qry_key === last_executed_key) return; @@ -163,46 +182,40 @@ async function handle_search_refresh(params: any) { $journals_sess.entry.qry__status = 'loading'; }); + if (remote_first) { + untrack(() => { + search_id_li = []; + }); + } + const qry_str = params.str; const cat_code = params.cat; + const order_by_li = { + group: 'DESC', + priority: 'DESC', + sort: 'DESC', + updated_on: 'DESC', + created_on: 'DESC' + }; let local_ids: string[] = []; // 3. FAST PATH: Local IDB Search (SWR) - // We skip this ONLY if remote_first is checked AND we have search text - if (!remote_first) { + // Broad views still need the local IDB set so "All" remains complete. + // Remote-first is only used to skip the local fast path for text searches. + if (!remote_first || !qry_str) { try { if (journal_id) { - let local_results = await db_journals.journal_entry + const local_results = await db_journals.journal_entry .where('journal_id') .equals(journal_id) - .filter((entry) => { - if (cat_code && entry.category_code !== cat_code) - return false; - if (qry_str) { - const name = (entry.name ?? '').toLowerCase(); - const content = (entry.content ?? '').toLowerCase(); - return ( - name.includes(qry_str) || - content.includes(qry_str) - ); - } - return true; - }) .toArray(); - local_results.sort((a, b) => { - const dateA = a.updated_on - ? new Date(a.updated_on).getTime() - : 0; - const dateB = b.updated_on - ? new Date(b.updated_on).getTime() - : 0; - return dateB - dateA; - }); + const filtered_results = + journal_entry_filter_list(local_results, params) ?? []; - local_ids = local_results - .map((e) => e.id || e.journal_entry_id) + local_ids = filtered_results + .map((entry) => entry.id || entry.journal_entry_id) .filter(Boolean); if (current_search_id === last_search_id) { @@ -233,21 +246,25 @@ async function handle_search_refresh(params: any) { enabled: params.enabled, hidden: params.hidden, limit: params.limit, + order_by_li, log_lvl: 0 }); if (current_search_id === last_search_id) { - const api_results = results || []; + const api_results = (results || []) as Array<{ + id?: string; + journal_entry_id?: string | null; + }>; const api_ids = api_results - .map((e: any) => e.id || e.journal_entry_id) - .filter(Boolean); + .map((entry) => entry.id || entry.journal_entry_id) + .filter((entry): entry is string => Boolean(entry)); + const display_ids = !qry_str && local_ids.length > 0 ? local_ids : api_ids; // Protect UI cache if API returns empty during revalidation if ( - api_ids.length === 0 && + !qry_str && local_ids.length > 0 && - !remote_first && - !qry_str + api_ids.length === 0 ) { untrack(() => { $journals_sess.entry.qry__status = 'done'; @@ -257,7 +274,7 @@ async function handle_search_refresh(params: any) { untrack(() => { $journals_sess.entry_li = api_results; - search_id_li = api_ids; + search_id_li = display_ids; $journals_sess.entry.qry__status = 'done'; }); if (log_lvl) diff --git a/src/routes/journals/ae_comp__journal_entry_obj_li.svelte b/src/routes/journals/ae_comp__journal_entry_obj_li.svelte index a1c88985..5578685a 100644 --- a/src/routes/journals/ae_comp__journal_entry_obj_li.svelte +++ b/src/routes/journals/ae_comp__journal_entry_obj_li.svelte @@ -84,26 +84,7 @@ let visible_journal_entry_obj_li = $derived( if (list === undefined || list === null) return null; if (!Array.isArray(list)) return []; - const filtered = list.filter((item: any) => { - if (!item) return false; - - const is_hidden = item.hide === true || item.hide === 1; - const is_disabled = item.enable === false || item.enable === 0; - - // Standard Visibility: Filter out hidden/disabled if not in Edit Mode - if (!$ae_loc.edit_mode) { - return !is_hidden && !is_disabled; - } - - // Edit Mode Gating: - // - To see Hidden: Must have Trusted Access or higher - if (is_hidden && !$ae_loc.trusted_access) return false; - - // - To see Disabled: Must have Administrator Access or higher - if (is_disabled && !$ae_loc.administrator_access) return false; - - return true; - }); + const filtered = list.filter((item: any) => !!item); if (log_lvl) console.log( @@ -125,7 +106,9 @@ let visible_journal_entry_obj_li = $derived(
- {#if visible_journal_entry_obj_li === null} + {#if visible_journal_entry_obj_li === null || + ($journals_sess.entry.qry__status === 'loading' && + visible_journal_entry_obj_li.length === 0)}
diff --git a/src/routes/journals/ae_comp__journal_entry_obj_qry.svelte b/src/routes/journals/ae_comp__journal_entry_obj_qry.svelte index f239f0fa..d025712e 100644 --- a/src/routes/journals/ae_comp__journal_entry_obj_qry.svelte +++ b/src/routes/journals/ae_comp__journal_entry_obj_qry.svelte @@ -1,72 +1,33 @@