From 48fc97cf46c7749e3bc235aed8a1786a87a5d2a0 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Fri, 20 Feb 2026 17:18:21 -0500 Subject: [PATCH] feat: add priority filtering and sort stability to V3 Lookup System --- app/methods/lookup_methods.py | 16 ++++----- app/routers/lookup_v3.py | 4 ++- documentation/GUIDE__V3_FRONTEND_API.md | 36 ++++++++++++++++++- .../PROJECT__V3_UNIFORM_LOOKUP_SYSTEM.md | 16 ++++++--- tests/e2e/test_e2e_v3_lookup.py | 19 ++++++++-- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/app/methods/lookup_methods.py b/app/methods/lookup_methods.py index e7d84cb..35a5488 100644 --- a/app/methods/lookup_methods.py +++ b/app/methods/lookup_methods.py @@ -12,22 +12,17 @@ def get_lookup_list_v3( for_type: Optional[str] = None, for_id: Optional[int] = None, include_disabled: bool = False, - whitelist: Optional[List[str]] = None + whitelist: Optional[List[str]] = None, + only_priority: bool = False ) -> List[dict]: """ Retrieves a ranked, deduplicated list of lookup records. Priority: Object Override > Account Override > Global Default. - Supports an optional whitelist (List of 'group' strings). + Supports an optional whitelist and priority filtering. """ table_name = f"v_lu_v3_{lu_type}" # We use ROW_NUMBER() to handle the hierarchy - # 1. Object specific (matching for_type and for_id) - # 2. Account specific (matching account_id) - # 3. Global (account_id IS NULL) - - # Whitelist logic: If a whitelist is provided, we only want records where - # the group is in that list. sql = f""" SELECT * FROM ( @@ -56,7 +51,10 @@ def get_lookup_list_v3( if not include_disabled: sql += " AND enable = 1" - sql += " ORDER BY sort ASC, name ASC" + if only_priority: + sql += " AND priority = 1" + + sql += " ORDER BY COALESCE(priority, 0) DESC, COALESCE(sort, 0) DESC, name ASC" params = { "account_id": account_ctx.account_id, diff --git a/app/routers/lookup_v3.py b/app/routers/lookup_v3.py index 2168e04..076387b 100644 --- a/app/routers/lookup_v3.py +++ b/app/routers/lookup_v3.py @@ -20,6 +20,7 @@ async def get_v3_lookup_list( for_id: Optional[int] = Query(None), site_id: Optional[str] = Query(None, min_length=8, max_length=22), include_disabled: bool = Query(False), + only_priority: bool = Query(False), account_ctx: AccountContext = Depends(get_account_context), response: Response = Response ): @@ -55,7 +56,8 @@ async def get_v3_lookup_list( for_type=for_type, for_id=for_id, include_disabled=include_disabled, - whitelist=whitelist + whitelist=whitelist, + only_priority=only_priority ) if not results and not include_disabled: diff --git a/documentation/GUIDE__V3_FRONTEND_API.md b/documentation/GUIDE__V3_FRONTEND_API.md index cf09dac..c342137 100644 --- a/documentation/GUIDE__V3_FRONTEND_API.md +++ b/documentation/GUIDE__V3_FRONTEND_API.md @@ -60,7 +60,41 @@ The primary way to retrieve data. --- -## 4. Event File Data Retrieval (Hosted Files) +## 4. V3 Uniform Lookup System + +The V3 Lookup system provides a hierarchical, deduplicated interface for standardized tables (Countries, Timezones, etc.). It supports global defaults, account overrides, and site-specific whitelisting. + +### A. List Lookups +Retrieve a ranked and filtered list of lookup items. +* **Endpoint:** `GET /v3/lookup/{lu_type}/list` +* **Available Types:** `country`, `country_subdivision`, `time_zone` +* **Parameters:** + * `site_id` (Optional): Random ID of the site to apply a **Whitelist Policy**. + * `only_priority` (Optional): Set to `true` to return only high-priority items (e.g., common time zones). + * `for_type` / `for_id` (Optional): Context for object-specific overrides. + * `include_disabled` (Optional): Set to `true` to see shadowed/disabled records. + +### B. Resolve Identity +Resolves a string (code, group, or name) to a single record. +* **Endpoint:** `GET /v3/lookup/{lu_type}/resolve?q=VALUE` +* **Usage:** Use this when you have an external code (e.g., ISO "US") and need the full Aether record. + +### C. Site Whitelist Policy +To limit lookups for a specific site, add a `lookup_policy` to the `site.cfg_json` field. +**Schema:** +```json +{ + "lookup_policy": { + "country": ["US", "CA", "GB"], + "time_zone": ["America/New_York"] + } +} +``` +*Note: Whitelist values must match the `group` field in the database.* + +--- + +## 5. Event File Data Retrieval (Hosted Files) Every Event File (`event_file`) **must** have a linked Hosted File (`hosted_file`). The Hosted File itself is a metadata record for binary content (files), which is accessed via separate Action endpoints (e.g., `/v3/action/hosted_file/download`). This API endpoint provides metadata about the associated hosted file. To retrieve this additional metadata: diff --git a/documentation/PROJECT__V3_UNIFORM_LOOKUP_SYSTEM.md b/documentation/PROJECT__V3_UNIFORM_LOOKUP_SYSTEM.md index 9d48c06..05d75a8 100644 --- a/documentation/PROJECT__V3_UNIFORM_LOOKUP_SYSTEM.md +++ b/documentation/PROJECT__V3_UNIFORM_LOOKUP_SYSTEM.md @@ -1,5 +1,5 @@ # Project: V3 Uniform Lookup & Identity Agnostic Resolution -> **Status:** 🏗️ Implementation (Phase 1 Complete - Feb 20, 2026) +> **Status:** 🏗️ Implementation (Phase 2 & Whitelist Complete - Feb 20, 2026) > **Goal:** Standardize all `lu_*` tables into a hierarchical, identity-agnostic system supporting Global Defaults, Account Overrides, and Object Overrides. ## 1. Executive Summary @@ -23,6 +23,7 @@ To ensure consistency across the ecosystem, all lookup tables will migrate towar - `description`: Detailed explanation. - `enable`: Binary (1 = Active, 0 = Disabled). *Crucial for Negative Overrides.* - `hide`: UI Visibility flag. +- `priority`: Boolean/TinyInt (1 = High priority, 0 = Normal). - `sort`: Ordering priority. - `created_on` / `updated_on`: Audit timestamps. @@ -72,7 +73,13 @@ To "delete" a global default for a specific account, the account creates an over ### 3.2 View Requirements - **Standard Joins:** All `v_lu_v3_*` views must join the `account` table to provide `account_id_random`. -- **Hybrid Naming:** (Proposed) Views may use `COALESCE` to provide a default `name` from specialized fields if the physical `name` column is empty. +- **Hybrid Naming:** Views may use `COALESCE` to provide a default `name` from specialized fields if the physical `name` column is empty. + +### 3.3 Whitelist Policy (Phase 2) +To avoid manual management of hundreds of negative overrides, accounts can define an opt-in whitelist. +- **Storage:** `site.cfg_json` -> `lookup_policy`. +- **Schema:** `{ "lookup_policy": { "country": ["US", "CA"] } }` +- **Logic:** When `site_id` is passed to the lookup API, results are filtered to only include groups present in the whitelist. --- @@ -89,19 +96,20 @@ To "delete" a global default for a specific account, the account creates an over - [x] `lu_v3_country_subdivision` (Group: `code`) - [x] `lu_v3_time_zone` (Group: `name`) -### Phase 3: Migration (Batch 2: Contextual) +### Phase 3: Migration (Batch 2: Contextual) - **ON HOLD FOR V3.1** - [ ] `lu_post_topic` - [ ] `lu_user_status` - [ ] `lu_file_purpose` ### Phase 4: Advanced Features -- [ ] Implement **Whitelist Policies** via `data_store` JSON. +- [x] Implement **Whitelist Policies** via `site.cfg_json`. - [ ] Implement Batch Update tools for managers. ## 5. Key Learnings & Decisions - **Deduplication:** The `PARTITION BY group` is the cornerstone of the system. Without a populated `group` field, the hierarchy collapses. - **Generic CRUD Compatibility:** We decided to maintain a physical `name` field to allow the Aether V3 UI to handle all lookups with zero custom code. - **Fail-Closed:** Account isolation is enforced at the query level via the `account_id` filter. +- **Type-Safe Auth:** Discovered that `load_site_obj` returns Random IDs for accounts; updated router context comparison to use `account_id_random` strings for 403 authorization checks. --- *Created by Gemini CLI for Scott Idem* diff --git a/tests/e2e/test_e2e_v3_lookup.py b/tests/e2e/test_e2e_v3_lookup.py index dfec129..7be1bd1 100644 --- a/tests/e2e/test_e2e_v3_lookup.py +++ b/tests/e2e/test_e2e_v3_lookup.py @@ -20,13 +20,16 @@ def print_result(label, success, message=""): status = "✅ PASS" if success else "❌ FAIL" print(f"{status} | {label} {': ' + message if message else ''}") -def test_lookup_list(lu_type, site_id=None): +def test_lookup_list(lu_type, site_id=None, only_priority=False): label = f"GET /{lu_type}/list" url = f"{BASE_URL}/{lu_type}/list" params = {} if site_id: params["site_id"] = site_id label += f" (Site: {site_id})" + if only_priority: + params["only_priority"] = "true" + label += " (Priority Only)" try: start_time = time.time() @@ -37,6 +40,16 @@ def test_lookup_list(lu_type, site_id=None): data = response.json().get('data', []) msg = f"Found {len(data)} items ({duration:.2f}s)" print_result(label, True, msg) + + # Print top 10 for sorting verification + if data and not site_id: # Only print for full or priority lists + limit = 10 if not only_priority else len(data) + print(f" Items:") + for i, item in enumerate(data[:limit]): + prio = item.get('priority', 0) + sort = item.get('sort', 0) + print(f" [{i+1}] {item.get('name')} (Prio: {prio}, Sort: {sort})") + return data else: print_result(label, False, f"Status {response.status_code}: {response.text[:100]}") @@ -68,7 +81,9 @@ if __name__ == "__main__": # 1. Basic Lists (Phase 1) test_lookup_list("country") - test_lookup_list("time_zone") + + print("\n--- Testing Priority Only ---") + test_lookup_list("time_zone", only_priority=True) # 2. Whitelist Test (Phase 2) if SITE_ID_RANDOM != "SET_ME_TO_SITE_ID":