Refine journal search filtering

This commit is contained in:
Scott Idem
2026-05-04 16:58:48 -04:00
parent 5cbdec3b5c
commit 285ef84b7e
9 changed files with 379 additions and 242 deletions

View File

@@ -306,6 +306,11 @@ These are real incidents — know them before you start.
redundant on the V3 path. Both paths now pretty-print with 2-space indent.
See `GUIDE__AE_API_V3_for_Frontend.md` → section 3C for the full explanation.
12. **Broad Dexie result windows get silently clipped** — if a broad "All" view shows fewer
rows than a narrower filter, check for a page-level limit or an API revalidation step
replacing the local IDB result set. For empty text searches, the full local result set
should drive the display; server refreshes should update cache, not shrink visibility.
---
## 8. Source Layout (Quick Reference)

View File

@@ -91,6 +91,7 @@ $effect(() => {
## Practical Patterns from Aether (Journals & Events)
- Journals: The journaling pages use SWR-style background refreshes but reliably render because either (a) the page `+page.ts` blocks to populate DB for critical views, or (b) components accept `data.initial_*` fallback values until `liveQuery` emits. This hybrid approach avoids the "refresh twice" problem while keeping navigation snappy.
- Journals broad views: if text search is empty, let the local IDB result set drive the visible list. The API can revalidate the cache in the background, but it should not replace a broad "All" view with a limited slice that hides valid rows.
- Sessions / Presentations: The session page demonstrates several best practices:
- Use `url_*` constants (derived from `data.params`) so the `liveQuery` closure captures a stable value instead of the reactive store directly.
@@ -113,6 +114,7 @@ let lq__event_presentation_obj_li = $derived(
- Add a small `console.log` inside each `liveQuery` closure to confirm when it runs and what `id` it sees.
- Verify that `+page.ts` either `await`s critical loads or returns `initial_*` payloads for first-render hydration.
- Confirm that dependent store values (selected IDs) are assigned before components subscribe — use `untrack` to prevent extra reactive cycles.
- If a broad Dexie-backed list shows fewer rows than a narrower filter, look for a limit or revalidation step overwriting the local IDB result set. Broad views should stay unbounded unless the user is actually narrowing by text.
- Ensure your `liveQuery` closures return quickly and do not throw; any exception inside the query can stop updates.
- If a dependent query appears stale, temporarily add `await 0` in the upstream query or an explicit `Promise.resolve()` after the IDB write to force the microtask queue to flush during debugging.

View File

@@ -0,0 +1,105 @@
export interface JournalEntrySearchParams {
str?: string;
cat?: string | null;
enabled?: 'enabled' | 'all' | 'not_enabled';
hidden?: 'hidden' | 'all' | 'not_hidden';
limit?: number;
}
function normalize_search_value(value: unknown): string {
if (value === null || value === undefined) return '';
if (typeof value === 'string') return value.toLowerCase();
if (typeof value === 'number' || typeof value === 'boolean') {
return String(value).toLowerCase();
}
try {
return JSON.stringify(value).toLowerCase();
} catch {
return String(value).toLowerCase();
}
}
function build_journal_entry_search_blob(entry: any): string {
return [
entry?.code,
entry?.name,
entry?.short_name,
entry?.summary,
entry?.outline,
entry?.content,
entry?.history,
entry?.notes,
entry?.tags,
entry?.activity_code,
entry?.category_code,
entry?.type_code,
entry?.topic_code,
entry?.group,
entry?.journal_code,
entry?.journal_name,
entry?.alert_msg,
entry?.default_qry_str
]
.map(normalize_search_value)
.filter(Boolean)
.join(' ');
}
export function journal_entry_matches_search(
entry: any,
params: JournalEntrySearchParams
): boolean {
if (!entry) return false;
const qry_str = (params.str ?? '').toLowerCase().trim();
const category_code = params.cat ?? '';
const enabled_mode = params.enabled ?? 'all';
const hidden_mode = params.hidden ?? 'all';
const is_hidden = entry.hide === true || entry.hide === 1;
const is_disabled = entry.enable === false || entry.enable === 0;
if (category_code && entry.category_code !== category_code) return false;
if (enabled_mode === 'enabled' && is_disabled) return false;
if (enabled_mode === 'not_enabled' && !is_disabled) return false;
if (hidden_mode === 'hidden' && !is_hidden) return false;
if (hidden_mode === 'not_hidden' && is_hidden) return false;
if (!qry_str) return true;
return build_journal_entry_search_blob(entry).includes(qry_str);
}
export function journal_entry_compare_for_list(a: any, b: any): number {
return (
(b?.tmp_sort_1 ?? '').localeCompare(a?.tmp_sort_1 ?? '') ||
(b?.updated_on ?? '').localeCompare(a?.updated_on ?? '') ||
(b?.journal_entry_id ?? '').localeCompare(a?.journal_entry_id ?? '')
);
}
export function journal_entry_filter_list(
list: any[] | null | undefined,
params: JournalEntrySearchParams
): any[] | null {
if (list === undefined || list === null) return null;
if (!Array.isArray(list)) return [];
const has_text_search = Boolean((params.str ?? '').trim());
const filtered = list
.filter((entry) => journal_entry_matches_search(entry, params))
.sort(journal_entry_compare_for_list);
// Broad views should show the full local result set; only text searches
// should be sliced to a page-sized window.
if (has_text_search && params.limit && params.limit > 0) {
return filtered.slice(0, params.limit);
}
return filtered;
}

View File

@@ -235,7 +235,7 @@ Middle-click to open in new tab`}>
</a>
{:else}
<!-- Edit Journal button. Creates a modal to edit the journal. -->
<Journal_entry_obj_qry {log_lvl} {lq__journal_obj} />
<Journal_entry_obj_qry {log_lvl} lq__journal_obj={$lq__journal_obj} />
{/if}
<!-- Add default journal entry -->

View File

@@ -3,7 +3,10 @@
let log_lvl: number = $state(0);
interface Props {
data: any;
data: {
account_id: string;
[key: string]: unknown;
};
}
let { data }: Props = $props();
@@ -16,23 +19,13 @@ import { untrack } from 'svelte';
import { liveQuery } from 'dexie';
// *** Import Aether specific variables and functions
import { ae_util } from '$lib/ae_utils/ae_utils';
import {
ae_snip,
ae_loc,
ae_sess,
ae_api,
ae_trig,
slct,
slct_trigger
} from '$lib/stores/ae_stores';
import { ae_loc, ae_api } from '$lib/stores/ae_stores';
import { db_journals } from '$lib/ae_journals/db_journals';
import { journal_entry_filter_list } from '$lib/ae_journals/ae_journals_search_helpers';
import {
journals_loc,
journals_sess,
journals_slct,
journals_prom,
journals_trig
journals_slct
} from '$lib/ae_journals/ae_journals_stores';
import { journals_func } from '$lib/ae_journals/ae_journals_functions';
@@ -42,12 +35,34 @@ import AeCompModalJournalExport from '../ae_comp__modal_journal_export.svelte';
import AeCompModalJournalImport from '../ae_comp__modal_journal_import.svelte';
// Variables
let ae_acct = $derived(data[data.account_id]);
interface JournalPageAccount {
api?: unknown;
loc?: {
person_id?: string | null;
};
slct: {
journal_id: string | null;
journal_entry_id?: string | null;
};
}
let ae_acct = $derived(data[data.account_id] as JournalPageAccount);
let show_export_modal = $state(false);
let show_import_modal = $state(false);
interface JournalSearchParams {
v: number;
str: string;
cat: string;
limit: number;
enabled: 'enabled' | 'all' | 'not_enabled';
hidden: 'hidden' | 'all' | 'not_hidden';
journal_id: string | null;
remote_first: boolean;
}
let search_id_li: Array<string> = $state([]);
let search_debounce_timer: any = null;
let search_debounce_timer: ReturnType<typeof setTimeout> | null = null;
let last_search_id = 0;
let last_executed_key = ''; // Search Guard Key
@@ -72,45 +87,6 @@ let lq__journal_obj = $derived(
})
);
// Stable LiveQuery Pattern (Aether UI V3)
// Re-wrapped in $derived to ensure the observable instance remains stable
// unless the underlying dependencies (ids, search context) change.
// Important: keep the `liveQuery` closure free of transient reactive
// references — capture stable values (ids, search keys) so the observable
// isn't recreated unnecessarily on every render. Use `search_id_li` or
// other plain arrays/values as explicit dependencies.
let lq__journal_entry_obj_li = $derived(
liveQuery(async () => {
const ids = search_id_li;
const journal_id = $lq__journal_obj?.journal_id;
const search_text = $journals_loc.entry.qry__search_text;
const cat_code = $journals_loc.entry.qry__category_code;
// SCENARIO 1: Specific IDs provided (Search Results)
if (Array.isArray(ids) && ids.length > 0) {
if (log_lvl)
console.log(`Journal Page LQ: bulkGet ${ids.length} IDs`);
const results = await db_journals.journal_entry.bulkGet(ids);
return results.filter((item) => item !== undefined);
}
// SCENARIO 2: Fallback to broad search (Default view)
if (journal_id && !search_text && !cat_code) {
if (log_lvl)
console.log(
`Journal Page LQ: Fallback search for journal: ${journal_id}`
);
return await db_journals.journal_entry
.where('journal_id')
.equals(journal_id)
.reverse()
.sortBy('tmp_sort_1');
}
return [];
})
);
// Standardized Reactive Search Pattern (Aether UI V3)
// 1. Isolate dependencies into a stable derived object
let search_params = $derived({
@@ -121,10 +97,53 @@ let search_params = $derived({
enabled: $journals_loc.entry.qry__enabled,
hidden: $journals_loc.entry.qry__hidden,
journal_id: $journals_slct.journal_id,
person_id: $ae_loc.person_id,
remote_first: $journals_loc.entry.qry__remote_first
});
// Stable LiveQuery Pattern (Aether UI V3)
// Re-wrapped in $derived to ensure the observable instance remains stable
// unless the underlying dependencies (ids, search context) change.
// Important: keep the `liveQuery` closure free of transient reactive
// references — capture stable values (ids, search keys) so the observable
// isn't recreated unnecessarily on every render. Use `search_id_li` or
// other plain arrays/values as explicit dependencies.
let lq__journal_entry_obj_li = $derived(
(() => {
const ids = search_id_li;
const params = search_params;
const journal_id = $lq__journal_obj?.journal_id;
return liveQuery(async () => {
if (params.remote_first && (!Array.isArray(ids) || ids.length === 0)) {
return null;
}
// SCENARIO 1: Specific IDs provided (Search Results)
if (Array.isArray(ids) && ids.length > 0) {
if (log_lvl)
console.log(`Journal Page LQ: bulkGet ${ids.length} IDs`);
const results = await db_journals.journal_entry.bulkGet(ids);
return results.filter((item) => item !== undefined);
}
if (!journal_id) return null;
// SCENARIO 2: Fallback to broad journal search (Default view)
if (log_lvl)
console.log(
`Journal Page LQ: Broad search for journal: ${journal_id}`
);
const results = await db_journals.journal_entry
.where('journal_id')
.equals(journal_id)
.toArray();
return journal_entry_filter_list(results, params) ?? [];
});
})()
);
// 2. Controlled effect for triggering searches
$effect(() => {
// Establishes reactive dependency on search_params
@@ -143,7 +162,7 @@ $effect(() => {
};
});
async function handle_search_refresh(params: any) {
async function handle_search_refresh(params: JournalSearchParams) {
// 1. Guard: Check if criteria actually changed
const qry_key = JSON.stringify(params);
if (qry_key === last_executed_key) return;
@@ -163,46 +182,40 @@ async function handle_search_refresh(params: any) {
$journals_sess.entry.qry__status = 'loading';
});
if (remote_first) {
untrack(() => {
search_id_li = [];
});
}
const qry_str = params.str;
const cat_code = params.cat;
const order_by_li = {
group: 'DESC',
priority: 'DESC',
sort: 'DESC',
updated_on: 'DESC',
created_on: 'DESC'
};
let local_ids: string[] = [];
// 3. FAST PATH: Local IDB Search (SWR)
// We skip this ONLY if remote_first is checked AND we have search text
if (!remote_first) {
// Broad views still need the local IDB set so "All" remains complete.
// Remote-first is only used to skip the local fast path for text searches.
if (!remote_first || !qry_str) {
try {
if (journal_id) {
let local_results = await db_journals.journal_entry
const local_results = await db_journals.journal_entry
.where('journal_id')
.equals(journal_id)
.filter((entry) => {
if (cat_code && entry.category_code !== cat_code)
return false;
if (qry_str) {
const name = (entry.name ?? '').toLowerCase();
const content = (entry.content ?? '').toLowerCase();
return (
name.includes(qry_str) ||
content.includes(qry_str)
);
}
return true;
})
.toArray();
local_results.sort((a, b) => {
const dateA = a.updated_on
? new Date(a.updated_on).getTime()
: 0;
const dateB = b.updated_on
? new Date(b.updated_on).getTime()
: 0;
return dateB - dateA;
});
const filtered_results =
journal_entry_filter_list(local_results, params) ?? [];
local_ids = local_results
.map((e) => e.id || e.journal_entry_id)
local_ids = filtered_results
.map((entry) => entry.id || entry.journal_entry_id)
.filter(Boolean);
if (current_search_id === last_search_id) {
@@ -233,21 +246,25 @@ async function handle_search_refresh(params: any) {
enabled: params.enabled,
hidden: params.hidden,
limit: params.limit,
order_by_li,
log_lvl: 0
});
if (current_search_id === last_search_id) {
const api_results = results || [];
const api_results = (results || []) as Array<{
id?: string;
journal_entry_id?: string | null;
}>;
const api_ids = api_results
.map((e: any) => e.id || e.journal_entry_id)
.filter(Boolean);
.map((entry) => entry.id || entry.journal_entry_id)
.filter((entry): entry is string => Boolean(entry));
const display_ids = !qry_str && local_ids.length > 0 ? local_ids : api_ids;
// Protect UI cache if API returns empty during revalidation
if (
api_ids.length === 0 &&
!qry_str &&
local_ids.length > 0 &&
!remote_first &&
!qry_str
api_ids.length === 0
) {
untrack(() => {
$journals_sess.entry.qry__status = 'done';
@@ -257,7 +274,7 @@ async function handle_search_refresh(params: any) {
untrack(() => {
$journals_sess.entry_li = api_results;
search_id_li = api_ids;
search_id_li = display_ids;
$journals_sess.entry.qry__status = 'done';
});
if (log_lvl)

View File

@@ -84,26 +84,7 @@ let visible_journal_entry_obj_li = $derived(
if (list === undefined || list === null) return null;
if (!Array.isArray(list)) return [];
const filtered = list.filter((item: any) => {
if (!item) return false;
const is_hidden = item.hide === true || item.hide === 1;
const is_disabled = item.enable === false || item.enable === 0;
// Standard Visibility: Filter out hidden/disabled if not in Edit Mode
if (!$ae_loc.edit_mode) {
return !is_hidden && !is_disabled;
}
// Edit Mode Gating:
// - To see Hidden: Must have Trusted Access or higher
if (is_hidden && !$ae_loc.trusted_access) return false;
// - To see Disabled: Must have Administrator Access or higher
if (is_disabled && !$ae_loc.administrator_access) return false;
return true;
});
const filtered = list.filter((item: any) => !!item);
if (log_lvl)
console.log(
@@ -125,7 +106,9 @@ let visible_journal_entry_obj_li = $derived(
</div>
<section
class="journal_list relative flex w-full flex-col items-center justify-center gap-1 md:gap-2">
{#if visible_journal_entry_obj_li === null}
{#if visible_journal_entry_obj_li === null ||
($journals_sess.entry.qry__status === 'loading' &&
visible_journal_entry_obj_li.length === 0)}
<!-- Loading state -->
<div
class="flex flex-col items-center justify-center p-10 opacity-50">

View File

@@ -1,72 +1,33 @@
<script lang="ts">
interface JournalObjLike {
name?: string;
cfg_json?: {
category_li?: Array<{
code: string;
name: string;
}>;
};
}
interface Props {
log_lvl?: number;
lq__journal_obj: any;
lq__journal_obj: JournalObjLike | null | undefined;
}
let { log_lvl = $bindable(0), lq__journal_obj }: Props = $props();
import {
ArrowDown01,
ArrowDown10,
ArrowDownUp,
BetweenVerticalEnd,
BetweenVerticalStart,
BookHeart,
BookImage,
Bookmark,
BookOpenText,
BriefcaseBusiness,
Check,
Copy,
Expand,
Eye,
EyeOff,
Flag,
FlagOff,
FilePlus,
Fingerprint,
Globe,
Library,
MessageSquareWarning,
Minus,
Notebook,
Pencil,
Plus,
RemoveFormatting,
SquareLibrary,
Shapes,
Share2,
ShieldCheck,
ShieldMinus,
Siren,
Skull,
Tags,
Target,
ToggleLeft,
ToggleRight,
Trash2,
TypeOutline,
X
RemoveFormatting
} from '@lucide/svelte';
import {
ae_snip,
ae_loc,
ae_sess,
ae_api,
ae_trig,
slct,
slct_trigger
} from '$lib/stores/ae_stores';
import {
journals_loc,
journals_sess,
journals_slct,
journals_prom,
journals_trig
journals_sess
} from '$lib/ae_journals/ae_journals_stores';
import { journals_func } from '$lib/ae_journals/ae_journals_functions';
// *** Functions and Logic
function handle_search_trigger() {
@@ -102,10 +63,10 @@ function prevent_default<T extends Event>(fn: (event: T) => void) {
type="text"
placeholder="Search Journal Entries"
bind:value={$journals_loc.entry.qry__search_text}
onkeyup={(event) => {
onkeyup={() => {
// Reactive effect in parent handles this debounced
}}
title={`Search for Entries in "${$lq__journal_obj?.name}. Press Enter to search.`}
title={`Search for Entries in "${lq__journal_obj?.name}. Press Enter to search.`}
autocomplete="off"
class="
input input-sm
@@ -153,17 +114,47 @@ function prevent_default<T extends Event>(fn: (event: T) => void) {
<select
class="select select-sm"
bind:value={$journals_loc.entry.qry__category_code}
onchange={(event) => {
onchange={() => {
handle_search_trigger();
}}
title="Filter by category">
<option value="">All Categories</option>
{#each $lq__journal_obj?.cfg_json?.category_li as category (category.code)}
{#each lq__journal_obj?.cfg_json?.category_li as category (category.code)}
<option value={category.code}>{category.name}</option>
{/each}
</select>
</span>
<span class="flex flex-row flex-wrap items-center gap-2">
<span class="hidden text-sm text-gray-500 lg:inline"> Filters: </span>
<label class="flex flex-row items-center gap-1 text-xs font-semibold text-gray-500">
<span>Enabled</span>
<select
class="select select-sm"
bind:value={$journals_loc.entry.qry__enabled}
onchange={handle_search_trigger}
title="Filter by enabled status">
<option value="enabled">Enabled Only</option>
<option value="not_enabled">Disabled Only</option>
<option value="all">All</option>
</select>
</label>
<label class="flex flex-row items-center gap-1 text-xs font-semibold text-gray-500">
<span>Hidden</span>
<select
class="select select-sm"
bind:value={$journals_loc.entry.qry__hidden}
onchange={handle_search_trigger}
title="Filter by hidden status">
<option value="not_hidden">Visible Only</option>
<option value="hidden">Hidden Only</option>
<option value="all">All</option>
</select>
</label>
</span>
<!-- Search Control Toggles -->
<span
class="border-surface-300-700 flex flex-row flex-wrap items-center gap-2 border-l pl-2">

View File

@@ -1,80 +1,110 @@
import { describe, it, expect } from 'vitest';
// Simulating the filter logic from ae_comp__journal_entry_obj_li.svelte
function filterEntries(list: any[], ae_loc: any) {
if (!list) return null;
return list.filter((item: any) => {
if (!item) return false;
import {
journal_entry_filter_list,
journal_entry_matches_search
} from '$lib/ae_journals/ae_journals_search_helpers';
const is_hidden = item.hide === true || item.hide === 1;
const is_disabled = item.enable === false || item.enable === 0;
// Standard Visibility: Filter out hidden/disabled if not in Edit Mode
if (!ae_loc.edit_mode) {
return !is_hidden && !is_disabled;
}
// Edit Mode Gating:
// - To see Hidden: Must have Trusted Access or higher
if (is_hidden && !ae_loc.trusted_access) return false;
// - To see Disabled: Must have Administrator Access or higher
if (is_disabled && !ae_loc.administrator_access) return false;
return true;
});
}
describe('Journal Entry Visibility Filtering', () => {
describe('Journal Entry Search Filtering', () => {
const mockEntries = [
{ id: '1', name: 'Normal Entry', hide: false, enable: true },
{ id: '2', name: 'Hidden Entry', hide: true, enable: true },
{ id: '3', name: 'Disabled Entry', hide: false, enable: false },
{ id: '4', name: 'Hidden & Disabled', hide: true, enable: false }
{
id: '4',
journal_entry_id: '4',
name: 'Hidden & Disabled',
hide: true,
enable: false,
tmp_sort_1: 'd'
},
{
id: '3',
journal_entry_id: '3',
name: 'Disabled Entry',
hide: false,
enable: false,
tmp_sort_1: 'c'
},
{
id: '2',
journal_entry_id: '2',
name: 'Hidden Entry',
hide: true,
enable: true,
tmp_sort_1: 'b'
},
{
id: '1',
journal_entry_id: '1',
name: 'Normal Entry',
summary: 'Alpha notes',
content: 'Beta details',
category_code: 'general',
hide: false,
enable: true,
tmp_sort_1: 'a'
}
];
it('should show only normal entries when Edit Mode is OFF (Manager)', () => {
const ae_loc = {
edit_mode: false,
trusted_access: true,
administrator_access: true
};
const result = filterEntries(mockEntries, ae_loc);
expect(result?.length).toBe(1);
expect(result?.[0].id).toBe('1');
});
it('should return all entries when filters are broad', () => {
const result = journal_entry_filter_list(mockEntries, {
str: '',
cat: '',
enabled: 'all',
hidden: 'all',
limit: 2
});
it('should show hidden entries to Trusted users when Edit Mode is ON', () => {
const ae_loc = {
edit_mode: true,
trusted_access: true,
administrator_access: false
};
const result = filterEntries(mockEntries, ae_loc);
// Should see Normal (1) and Hidden (2). Should NOT see Disabled (3, 4)
expect(result?.length).toBe(2);
expect(result?.map((r) => r.id)).toContain('1');
expect(result?.map((r) => r.id)).toContain('2');
});
it('should show everything to Administrators when Edit Mode is ON', () => {
const ae_loc = {
edit_mode: true,
trusted_access: true,
administrator_access: true
};
const result = filterEntries(mockEntries, ae_loc);
expect(result?.length).toBe(4);
expect(result?.map((entry) => entry.id)).toEqual([
'4',
'3',
'2',
'1'
]);
});
it('should hide everything sensitive to Public users even if Edit Mode is ON (Safety Check)', () => {
const ae_loc = {
edit_mode: true,
trusted_access: false,
administrator_access: false
};
const result = filterEntries(mockEntries, ae_loc);
it('should filter by enabled and hidden status', () => {
const result = journal_entry_filter_list(mockEntries, {
str: '',
cat: '',
enabled: 'enabled',
hidden: 'not_hidden',
limit: 50
});
expect(result?.length).toBe(1);
expect(result?.[0].id).toBe('1');
});
it('should match text across summary and content', () => {
expect(
journal_entry_matches_search(mockEntries[3], {
str: 'alpha',
cat: '',
enabled: 'all',
hidden: 'all'
})
).toBe(true);
expect(
journal_entry_matches_search(mockEntries[3], {
str: 'beta',
cat: '',
enabled: 'all',
hidden: 'all'
})
).toBe(true);
});
it('should limit text-filtered results after sorting', () => {
const result = journal_entry_filter_list(mockEntries, {
str: 'entry',
cat: '',
enabled: 'all',
hidden: 'all',
limit: 2
});
expect(result?.length).toBe(2);
expect(result?.map((entry) => entry.id)).toEqual(['3', '2']);
});
});

View File

@@ -1,8 +1,12 @@
import { defineConfig } from 'vitest/config';
import { defineConfig, mergeConfig } from 'vitest/config';
import viteConfig from './vite.config';
export default defineConfig({
test: {
include: ['src/**/*.{test,spec}.ts'],
exclude: ['node_modules', 'tests']
}
});
export default mergeConfig(
viteConfig,
defineConfig({
test: {
include: ['src/**/*.{test,spec}.ts'],
exclude: ['node_modules', 'tests']
}
})
);