From 7db937f8afb52673ae9ad1060b3306e317be7e23 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Mon, 19 Jan 2026 15:57:00 -0500 Subject: [PATCH] Vision ID: Standardize Site Domain and Journal objects with string-only IDs and searchable mapping --- GEMINI.md | 16 +++ app/lib_api_crud_v3.py | 48 +++++--- app/lib_sql_crud.py | 61 +++++----- app/lib_sql_search.py | 53 ++++++--- app/main.py | 24 +++- app/models/journal_entry_models.py | 83 ++++++------- app/models/journal_models.py | 96 ++++++--------- app/models/site_domain_models.py | 183 ++++++++++------------------- app/object_definitions/cms.py | 5 +- app/object_definitions/journals.py | 8 +- 10 files changed, 272 insertions(+), 305 deletions(-) diff --git a/GEMINI.md b/GEMINI.md index fad4a77..db0e155 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -56,3 +56,19 @@ This session focused on infrastructure automation and resolving critical environ - **Storage**: Critical assets at `/home/scott/OSIT/hosted_files/` (Synced via Syncthing). - **Agents Sync**: Shared documentation and notifications pushed to `~/agents_sync/`. - **Coding Standards**: Adhering to `prompts/coding_standards.md`. + + +## 📋 Aether API Development Protocol + +0. Pre-Flight Check: Verify git status. Ensure all previous working changes are committed to maintain a "known good" state before starting a new cycle. +1. Strategic Plan: Write a concise plan identifying the issue, the specific files to be modified, and the verification steps (including specific curl commands or test scripts). +2. Implementation: Perform atomic code modifications using the replace or write_file tools. +3. Syntax Validation: Run python3 -m py_compile immediately. If syntax errors exist, they must be fixed before proceeding to deployment. +4. Process Cycle: Restart the Docker FastAPI service to apply changes: + * "docker restart aether_container_env-ae_api-2" +5. Empirical Testing: + * Execute the curl commands or test scripts identified in Step 1. + * Inspect the live log stream for regressions: + + 1 "tail -n 20 ~/OSIT_dev/aether_container_env/logs/ae_api/aether_api.log" +6. Finalize: Once verified, commit the changes with a descriptive message and sync relevant documentation. \ No newline at end of file diff --git a/app/lib_api_crud_v3.py b/app/lib_api_crud_v3.py index 3e321a9..cf81606 100644 --- a/app/lib_api_crud_v3.py +++ b/app/lib_api_crud_v3.py @@ -126,9 +126,10 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None """ Sanitizes an input payload before database insertion or update. - 1. Resolves virtual lookup fields (`*_id_random`) into their integer database IDs. - 2. Removes virtual lookup fields (ending in `_id_random`) that are used for API - convenience but do not exist in the database. + 1. Resolves ID strings to integers: + - Handles legacy `*_id_random` fields. + - Handles Vision `*_id` fields where the value is a string (e.g., account_id: "random_str"). + 2. Removes virtual lookup fields (ending in `_id_random`) after resolution. 3. Removes fields explicitly marked for exclusion in the model's `fields_to_exclude_from_db` ClassVar (e.g., view-only fields). 4. If `ignore_extra` is True, removes all fields NOT present in the model definition. @@ -140,22 +141,35 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None from app.db_sql import redis_lookup_id_random - # Resolve virtual _id_random fields to integer IDs (e.g., account_id_random -> account_id) - # This must happen BEFORE we delete them. + # Resolve ID strings to integers for k, v in list(data.items()): - if k.endswith('_id_random') and k != 'id_random' and v: + if not v or not isinstance(v, str): + continue + + target_id_field = None + obj_type_lookup = None + + # Scenario A: Legacy suffix (e.g., account_id_random: "abc") + if k.endswith('_id_random') and k != 'id_random': target_id_field = k.replace('_id_random', '_id') - # Only resolve if the integer version is missing or null - if not data.get(target_id_field): - obj_type_lookup = k.replace('_id_random', '') - resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup) - if resolved_id: - data[target_id_field] = resolved_id - - # Filter out virtual _id_random fields (e.g., account_id_random) - keys_to_remove = [k for k in data.keys() if k.endswith('_id_random') and k != 'id_random'] - for k in keys_to_remove: - del data[k] + obj_type_lookup = k.replace('_id_random', '') + + # Scenario B: Vision naming (e.g., account_id: "abc") + # We only resolve if it's a string of the correct length (random ID format) + elif k.endswith('_id') and 11 <= len(v) <= 22: + target_id_field = k + obj_type_lookup = k.replace('_id', '') + + if target_id_field and obj_type_lookup: + # Special table mapping if needed + if obj_type_lookup == 'address_location': obj_type_lookup = 'address' + + resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup) + if resolved_id: + data[target_id_field] = resolved_id + # If we were handling Scenario A, remove the original random key + if k.endswith('_id_random'): + del data[k] # Filter out model-specific excluded fields (e.g., view-only fields) if hasattr(model, 'fields_to_exclude_from_db'): diff --git a/app/lib_sql_crud.py b/app/lib_sql_crud.py index 730ecb1..19ddb2c 100644 --- a/app/lib_sql_crud.py +++ b/app/lib_sql_crud.py @@ -9,8 +9,9 @@ from sqlalchemy import text, Time from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError from app.log import log, logger_reset -# CRITICAL: Import the global connection state from lib_sql_core -from app.lib_sql_core import db, sql_connect, set_last_sql_error +# CRITICAL: Import the core module to access current global state +from app import lib_sql_core +from app.lib_sql_core import sql_connect, set_last_sql_error # Helper for resolving random IDs from app.lib_redis_helpers import lookup_id_random_pop @@ -50,9 +51,9 @@ def sql_insert( log.error('SQL INSERT statement could not be created. Missing params.') return False - trans = db.begin() + trans = lib_sql_core.db.begin() try: - result_insert = db.execute(sql_insert_stmt, data) + result_insert = lib_sql_core.db.execute(sql_insert_stmt, data) trans.commit() except IntegrityError as e: trans.rollback() @@ -122,16 +123,16 @@ def sql_update( else: return False - trans = db.begin() + trans = lib_sql_core.db.begin() try: - result_update = db.execute(sql_update_stmt, data) + result_update = lib_sql_core.db.execute(sql_update_stmt, data) trans.commit() except OperationalError: trans.rollback() log.error('Operational error (gone away?). Retrying once...') - sql_connect(current_db=db) + sql_connect(current_db=lib_sql_core.db) try: - result_update = db.execute(sql_update_stmt, data) + result_update = lib_sql_core.db.execute(sql_update_stmt, data) trans.commit() except Exception as e: set_last_sql_error(e) @@ -182,9 +183,9 @@ def sql_insert_or_update( else: return False - trans = db.begin() + trans = lib_sql_core.db.begin() try: - res = db.execute(stmt, data) + res = lib_sql_core.db.execute(stmt, data) trans.commit() return res.lastrowid if res.lastrowid > 0 else True except Exception as e: @@ -300,38 +301,32 @@ def sql_select( # ### END ### Core Help CRUD ### sql_select() ### -# ### BEGIN ### Core Help CRUD ### run_sql_select() ### +# ### BEGIN ### API DB SQL ### run_sql_select() ### @logger_reset def run_sql_select( - sql: Any, + sql: text, data: dict|None = None, - commit: bool = False, log_lvl: int = logging.WARNING, ) -> Any: log.setLevel(log_lvl) - if not db: - return False - + + print(f"Executing SQL: {sql} with data: {data}", flush=True) + try: - if commit: trans = db.begin() - sql = sql.columns(recurring_start_time=Time, recurring_end_time=Time) - result = db.execute(sql, data) if data else db.execute(sql) - if commit: trans.commit() - return result - except (OperationalError, ProgrammingError): - log.error('DB Error. Retrying once...') - sql_connect(current_db=db) + return lib_sql_core.db.execute(sql, data) + except (OperationalError, ProgrammingError) as e: + log.error(f'DB Error: {e}. Retrying once...') + sql_connect(current_db=lib_sql_core.db) try: - if commit: trans = db.begin() - result = db.execute(sql, data) if data else db.execute(sql) - if commit: trans.commit() - return result - except Exception: - return False + return lib_sql_core.db.execute(sql, data) + except Exception as e2: + set_last_sql_error(e2) + raise e2 # RAISING instead of returning False except Exception as e: log.exception(e) - return False -# ### END ### Core Help CRUD ### run_sql_select() ### + set_last_sql_error(e) + raise e # RAISING instead of returning False +# ### END ### API DB SQL ### run_sql_select() ### # ### BEGIN ### Core Help CRUD ### sql_delete() ### @logger_reset @@ -360,7 +355,7 @@ def sql_delete( return False try: - result = db.execute(stmt, data) if data else db.execute(stmt) + result = lib_sql_core.db.execute(stmt, data) if data else lib_sql_core.db.execute(stmt) return True if result.rowcount >= 1 else None except Exception as e: log.exception(e) diff --git a/app/lib_sql_search.py b/app/lib_sql_search.py index 27739f9..5263c5e 100644 --- a/app/lib_sql_search.py +++ b/app/lib_sql_search.py @@ -78,12 +78,12 @@ def sql_fulltext_qry_part(fulltext_qry_dict: dict) -> tuple[str, dict]|bool: def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool: """Handles enabled/disabled status filtering with schema check.""" - from app.db_sql import db + from app import lib_sql_core if not table_name: return False if enabled in ['enabled', 'disabled', 'all']: if enabled == 'all': return '', None try: - db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0")) + lib_sql_core.db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0")) except: log.warning(f"Table '{table_name}' missing 'enable' column. Skipping filter.") return '', None @@ -93,12 +93,12 @@ def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool def sql_hidden_part(table_name: str, hidden: str) -> tuple[str, bool|None]|bool: """Handles hidden status filtering with schema check.""" - from app.db_sql import db + from app import lib_sql_core if not table_name: return False if hidden in ['hidden', 'not_hidden', 'all']: if hidden == 'all': return '', None try: - db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0")) + lib_sql_core.db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0")) except: log.warning(f"Table '{table_name}' missing 'hide' column. Skipping filter.") return '', None @@ -133,7 +133,7 @@ def sql_search_qry_part( table_name: str|None = None, ) -> tuple[str, dict]: """Recursively builds a SQL WHERE clause from a SearchQuery model.""" - from app.db_sql import db + from app import lib_sql_core data = {} param_counter = [0] @@ -157,9 +157,13 @@ def sql_search_qry_part( else: use_match = True if table_name: - try: db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0")) - except: use_match = False - else: use_match = False + try: + lib_sql_core.db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0")) + except: + use_match = False + else: + use_match = False + if use_match: p_name = get_param_name() clauses.append(f"MATCH( default_qry_str ) AGAINST( :{p_name} IN BOOLEAN MODE )") @@ -172,7 +176,7 @@ def sql_search_qry_part( 'created_on', 'updated_on' ] for field in searchable_fields: - # Exclude exact internal integer IDs (ending in _id) + # Exclude internal integer IDs specifically if field.endswith('_id') or field == 'id': continue @@ -183,6 +187,7 @@ def sql_search_qry_part( f_p_name = get_param_name() like_clauses.append(f"`{field}` LIKE :{f_p_name}") data[f_p_name] = f"%{query_node.query_string}%" + if like_clauses: clauses.append(f"({' OR '.join(like_clauses)})") for filter_attr in ['and_filters', 'or_filters']: if hasattr(query_node, filter_attr) and getattr(query_node, filter_attr): @@ -198,19 +203,33 @@ def sql_search_qry_part( return ' AND '.join(clauses) def process_filter(f) -> tuple[str, dict]: - if searchable_fields is not None and f.field not in searchable_fields: - raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}'") + # --- ID VISION MAPPING --- + # If the frontend uses clean names (id, account_id), + # map them to the database columns (id_random, account_id_random). + target_field = f.field + vision_fields = ['id', 'account_id', 'site_id', 'person_id', 'user_id', 'journal_id', 'journal_entry_id'] + + if target_field in vision_fields: + if target_field == 'id': target_field = 'id_random' + else: target_field = f"{target_field}_random" + print(f"Search Trace: Mapping filter field '{f.field}' -> '{target_field}'", flush=True) + + if searchable_fields is not None and target_field not in searchable_fields: + # Fallback check for original field just in case + if f.field not in searchable_fields: + raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}' (mapped to '{target_field}')") + sql_op = operator_map.get(f.op.lower()) if not sql_op: raise HTTPException(status_code=400, detail=f"Unsupported operator: {f.op}") filter_data = {} - if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{f.field}` {sql_op}" + if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{target_field}` {sql_op}" else: p_name = get_param_name() - if f.op.lower() == 'in': clause = f"`{f.field}` IN (:{p_name})"; filter_data[p_name] = f.value - elif f.op.lower() in ['contains', 'icontains']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%" - elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%" - elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}" - else: clause = f"`{f.field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value + if f.op.lower() == 'in': clause = f"`{target_field}` IN (:{p_name})"; filter_data[p_name] = f.value + elif f.op.lower() in ['contains', 'icontains']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%" + elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%" + elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}" + else: clause = f"`{target_field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value return clause, filter_data sql_where = process_node(search_query, 1) diff --git a/app/main.py b/app/main.py index f620c89..06dc331 100644 --- a/app/main.py +++ b/app/main.py @@ -46,6 +46,14 @@ async def lifespan(app: FastAPI): # 2. Bootstrapping Configuration from DB with robust error handling log.info("Bootstrapping Configuration...") + + # Save original settings for fallback + orig_db_server = config.settings.DB_SERVER + orig_db_user = config.settings.DB_USER + orig_db_pass = config.settings.DB_PASS + orig_db_name = config.settings.DB_NAME + orig_db_port = config.settings.DB_PORT + try: if bootstrap_db_config(config.settings): log.info("Successfully bootstrapped configuration from database.") @@ -53,13 +61,25 @@ async def lifespan(app: FastAPI): if reconnect_db(): log.info("Database connection re-established with production configuration.") else: - log.warning("FAILED to re-establish database connection after bootstrap. Falling back to .env settings.") + log.warning("FAILED to re-establish database connection after bootstrap. Reverting to .env settings.") + config.settings.DB_SERVER = orig_db_server + config.settings.DB_USER = orig_db_user + config.settings.DB_PASS = orig_db_pass + config.settings.DB_NAME = orig_db_name + config.settings.DB_PORT = orig_db_port + reconnect_db() else: log.warning("System bootstrap from DB returned no results. Using environment defaults.") except Exception as e: log.error(f"Unexpected error during configuration bootstrap: {e}. Falling back to .env settings.") + config.settings.DB_SERVER = orig_db_server + config.settings.DB_USER = orig_db_user + config.settings.DB_PASS = orig_db_pass + config.settings.DB_NAME = orig_db_name + config.settings.DB_PORT = orig_db_port + reconnect_db() - # 2. Final validation of critical infrastructure + # 3. Final validation of critical infrastructure validate_critical_config(config.settings) log.info('### **** *** ** * Aether API v4 using FastAPI - Startup Sequence Complete * ** *** **** ###') diff --git a/app/models/journal_entry_models.py b/app/models/journal_entry_models.py index 3bc0766..db0bb88 100644 --- a/app/models/journal_entry_models.py +++ b/app/models/journal_entry_models.py @@ -1,7 +1,7 @@ import datetime, pytz from typing import Dict, List, Optional, Set, Union, ClassVar -from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator +from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator from app.db_sql import redis_lookup_id_random from app.lib_general import log, logging @@ -14,23 +14,18 @@ class Journal_Entry_Base(BaseModel): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['journal_entry_id_random'], - alias = 'journal_entry_id_random', - ) - id: Optional[int] = Field( - alias = 'journal_entry_id' - ) - - journal_id_random: Optional[str] - journal_id: Optional[int] + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['journal_entry_id_random']) + journal_entry_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random']) + journal_id: Optional[str] = Field(None, **base_fields['journal_id_random']) + account_id: Optional[str] = Field(None, **base_fields['account_id_random']) external_id: Optional[str] # ID generated by or for external systems (should be stable and not change) import_id: Optional[str] # Used for import purposes to track the source of the data code: Optional[str] for_type: Optional[str] # 'person', 'user', 'account', etc - for_id: Optional[int] + for_id: Optional[int] = Field(None, exclude=True) for_id_random: Optional[str] name: Optional[str] @@ -76,10 +71,11 @@ class Journal_Entry_Base(BaseModel): personal: Optional[bool] = True professional: Optional[bool] = False - parent_id_random: Optional[str] - parent_id: Optional[int] + parent_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random']) + # parent_id_random: Optional[str] + related_entry_id_random: Optional[List[str]] - related_entry_id_li: Optional[List[int]] + related_entry_id_li: Optional[List[int]] = Field(None, exclude=True) due_datetime: Optional[datetime.datetime] due_alert: Optional[bool] @@ -113,41 +109,38 @@ class Journal_Entry_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def journal_entry_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry') - return None - - @validator('journal_id', always=True) - def journal_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('journal_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='journal') - return None - - @validator('parent_id', always=True) - def parent_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('parent_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry') - return None - - @validator('for_id', always=True) - def for_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values.get('for_id_random') and values.get('for_type'): - return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type']) - return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + """ + Vision Transformer: + Map DB keys to clean API keys and strip internal integers. + """ + # 1. Map Random Strings to Clean Names + if rid := values.get('id_random') or values.get('journal_entry_id_random'): + values['id'] = rid + values['journal_entry_id'] = rid + + if j_rid := values.get('journal_id_random'): + values['journal_id'] = j_rid + + if a_rid := values.get('account_id_random'): + values['account_id'] = a_rid + + if p_rid := values.get('parent_id_random'): + values['parent_id'] = p_rid + + # 2. Prevent "Collision Population" + for k in ['id', 'journal_id', 'account_id', 'parent_id']: + if k in values and not isinstance(values[k], str): + del values[k] + + return values # Fields that are part of the model (for reading) but should not be saved to the DB table fields_to_exclude_from_db: ClassVar[list] = ['file_count'] class Config: underscore_attrs_are_private = True - allow_population_by_field_name = True + allow_population_by_field_name = False fields = base_fields # ### END ### API Journal Entry Models ### Journal_Entry_Base() ### diff --git a/app/models/journal_models.py b/app/models/journal_models.py index 513e203..1fcb723 100644 --- a/app/models/journal_models.py +++ b/app/models/journal_models.py @@ -1,7 +1,7 @@ import datetime, pytz from typing import Dict, List, Optional, Set, Union, ClassVar -from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator +from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator from app.db_sql import redis_lookup_id_random from app.lib_general import log, logging @@ -16,28 +16,19 @@ class Journal_Base(BaseModel): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['journal_id_random'], - alias = 'journal_id_random', - ) - id: Optional[int] = Field( - alias = 'journal_id' - ) - account_id_random: Optional[str] - account_id: Optional[int] - - person_id_random: Optional[str] - person_id: Optional[int] - - user_id_random: Optional[str] - user_id: Optional[int] + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['journal_id_random']) + journal_id: Optional[str] = Field(None, **base_fields['journal_id_random']) + account_id: Optional[str] = Field(None, **base_fields['account_id_random']) + person_id: Optional[str] = Field(None, **base_fields['person_id_random']) + user_id: Optional[str] = Field(None, **base_fields['user_id_random']) external_id: Optional[str] # ID generated by or for external systems (should be stable and not change) import_id: Optional[str] # Used for import purposes to track the source of the data code: Optional[str] for_type: Optional[str] # 'person', 'user', 'account', etc - for_id: Optional[int] + for_id: Optional[int] = Field(None, exclude=True) for_id_random: Optional[str] name: Optional[str] @@ -137,51 +128,30 @@ class Journal_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def journal_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='journal') - return None - - @validator('account_id', always=True) - def account_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('account_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='account') - return None - - @validator('person_id', always=True) - def person_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('person_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='person') - return None - - @validator('user_id', always=True) - def user_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('user_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='user') - return None - - @validator('for_id', always=True) - def for_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values.get('for_id_random') and values.get('for_type'): - return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type']) - return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + """ + Vision Transformer: + Map DB keys to clean API keys and strip internal integers. + """ + # 1. Map Random Strings to Clean Names + if rid := values.get('id_random') or values.get('journal_id_random'): + values['id'] = rid + values['journal_id'] = rid + + if a_rid := values.get('account_id_random'): + values['account_id'] = a_rid + if p_rid := values.get('person_id_random'): + values['person_id'] = p_rid + if u_rid := values.get('user_id_random'): + values['user_id'] = u_rid + + # 2. Prevent "Collision Population" + for k in ['id', 'account_id', 'person_id', 'user_id']: + if k in values and not isinstance(values[k], str): + del values[k] + + return values # Fields that are part of the model (for reading) but should not be saved to the DB table fields_to_exclude_from_db: ClassVar[list] = [ @@ -192,6 +162,6 @@ class Journal_Base(BaseModel): class Config: underscore_attrs_are_private = True - allow_population_by_field_name = True + allow_population_by_field_name = False fields = base_fields # ### END ### API Journal Models ### Journal_Base() ### diff --git a/app/models/site_domain_models.py b/app/models/site_domain_models.py index 7148a77..391978b 100644 --- a/app/models/site_domain_models.py +++ b/app/models/site_domain_models.py @@ -1,7 +1,7 @@ import datetime, pytz from typing import Dict, List, Optional, Set, Union -from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator +from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator from app.db_sql import get_id_random, redis_lookup_id_random from app.lib_general import log, logging @@ -14,16 +14,11 @@ class Site_Domain_Base(BaseModel): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['site_domain_id_random'], - alias = 'site_domain_id_random', - ) - id: Optional[int] = Field( - alias = 'site_domain_id' - ) - - site_id_random: Optional[str] - site_id: Optional[int] + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['site_domain_id_random']) + site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random']) + site_id: Optional[str] = Field(None, **base_fields['site_id_random']) + account_id: Optional[str] = Field(None, **base_fields['account_id_random']) fqdn: Optional[str] @@ -38,59 +33,45 @@ class Site_Domain_Base(BaseModel): cfg_json: Optional[Union[Json, None]] # In use 2024-03-04 - hide: Optional[bool] - priority: Optional[bool] - sort: Optional[int] - group: Optional[str] + hide: Optional[bool] = None # Field missing in physical table but common in views + # priority: Optional[bool] # MISSING in physical table + # sort: Optional[int] # MISSING in physical table + # group: Optional[str] # MISSING in physical table - notes: Optional[str] + notes: Optional[str] = None # MISSING in physical table created_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None - # Including convenience data - # This is only for convenience. Probably going to keep unless it causes a problem. - account_id: Optional[int] - account_id_random: Optional[str] - _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def site_domain_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain') - return None - - @validator('account_id', always=True) - def account_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('account_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='account') - return None - - @validator('account_id_random', always=True) - def account_id_random_lookup(cls, v, values, **kwargs): - if isinstance(v, str) and len(v) >= 11: return v - elif account_id := values.get('account_id'): - return get_id_random(record_id=account_id, table_name='account') - return None - - @validator('site_id', always=True) - def site_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['site_id_random']: - return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site') - return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + """ + Vision Transformer: + Map DB-centric keys to clean API keys and strip internal integers. + """ + # 1. Map Random Strings to Clean Names + # We prioritize the random strings to ensure the Vision is string-based. + if rid := values.get('id_random') or values.get('site_domain_id_random'): + values['id'] = rid + values['site_domain_id'] = rid + + if s_rid := values.get('site_id_random'): + values['site_id'] = s_rid + + if a_rid := values.get('account_id_random'): + values['account_id'] = a_rid + + # 2. Prevent "Collision Population" + for k in ['id', 'site_id', 'account_id']: + if k in values and not isinstance(values[k], str): + del values[k] + + return values class Config: underscore_attrs_are_private = True - fields = base_fields - allow_population_by_field_name = True + allow_population_by_field_name = False # ### END ### API Site Domain Models ### Site_Domain_Base() ### @@ -99,19 +80,11 @@ class Site_Domain_FQDN_ID_Base(BaseModel): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['site_domain_id_random'], - alias = 'site_domain_id_random', - ) - id: Optional[int] = Field( - alias = 'site_domain_id' - ) - - site_id_random: Optional[str] - site_id: Optional[int] - - site_domain_id_random: Optional[str] - site_domain_id: Optional[int] + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['site_domain_id_random']) + site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random']) + site_id: Optional[str] = Field(None, **base_fields['site_id_random']) + account_id: Optional[str] = Field(None, **base_fields['account_id_random']) fqdn: Optional[str] @@ -124,19 +97,13 @@ class Site_Domain_FQDN_ID_Base(BaseModel): valid_for: Optional[int] # number of hours enable: Optional[bool] - hide: Optional[bool] - priority: Optional[bool] - sort: Optional[int] - group: Optional[str] - - notes: Optional[str] + hide: Optional[bool] = None + + notes: Optional[str] = None created_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None # Including convenience data - # This is only for convenience. Probably going to keep unless it causes a problem. - account_id: Optional[int] - account_id_random: Optional[str] account_code: Optional[str] # Useful for export file naming account_name: Optional[str] # Generally useful for display account_enable: Optional[bool] @@ -157,54 +124,22 @@ class Site_Domain_FQDN_ID_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain') - return None - - @validator('site_domain_id_random', always=True) - def site_domain_id_random_lookup(cls, v, values, **kwargs): - if isinstance(v, str) and len(v) >= 11: return v - elif id_random := values.get('id_random'): - return id_random - return None - - @validator('account_id', always=True) - def account_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('account_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='account') - return None - - @validator('account_id_random', always=True) - def account_id_random_lookup(cls, v, values, **kwargs): - if isinstance(v, str) and len(v) >= 11: return v - elif account_id := values.get('account_id'): - return get_id_random(record_id=account_id, table_name='account') - return None - - @validator('site_id', always=True) - def site_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['site_id_random']: - return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site') - return None - - @validator('site_domain_id', always=True) - def site_domain_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['site_domain_id_random']: - return redis_lookup_id_random(record_id_random=values['site_domain_id_random'], table_name='site') - return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + if rid := values.get('id_random') or values.get('site_domain_id_random'): + values['id'] = rid + values['site_domain_id'] = rid + if s_rid := values.get('site_id_random'): + values['site_id'] = s_rid + if a_rid := values.get('account_id_random'): + values['account_id'] = a_rid + + for k in ['id', 'site_id', 'account_id']: + if k in values and not isinstance(values[k], str): + del values[k] + return values class Config: underscore_attrs_are_private = True - fields = base_fields - allow_population_by_field_name = True + allow_population_by_field_name = False # ### END ### API Site Domain Models ### Site_Domain_FQDN_ID_Base() ### diff --git a/app/object_definitions/cms.py b/app/object_definitions/cms.py index b76b856..171b923 100644 --- a/app/object_definitions/cms.py +++ b/app/object_definitions/cms.py @@ -116,8 +116,9 @@ cms_obj_li = { 'base_name_alt': Site_Domain_FQDN_ID_Base, # V3 Search Security: 'searchable_fields': [ - 'site_domain_id_random', 'account_id_random', 'site_id_random', - 'fqdn', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on' + 'id', 'account_id', 'site_id', + 'id_random', 'account_id_random', 'site_id_random', + 'fqdn', 'enable', 'created_on', 'updated_on' ], }, } diff --git a/app/object_definitions/journals.py b/app/object_definitions/journals.py index 683f742..59833e2 100644 --- a/app/object_definitions/journals.py +++ b/app/object_definitions/journals.py @@ -20,7 +20,9 @@ journal_obj_li = { ], # V3 Search Security: 'searchable_fields': [ - 'journal_id_random', 'name', 'short_name', 'summary', 'outline', + 'id', 'account_id', 'person_id', 'user_id', + 'journal_id_random', 'account_id_random', 'person_id_random', 'user_id_random', + 'name', 'short_name', 'summary', 'outline', 'description', 'type_code', 'tags', 'billable', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on' ], @@ -42,7 +44,9 @@ journal_obj_li = { ], # V3 Search Security: 'searchable_fields': [ - 'journal_entry_id_random', 'journal_id_random', 'name', 'short_name', + 'id', 'journal_id', 'account_id', + 'journal_entry_id_random', 'journal_id_random', 'account_id_random', + 'name', 'short_name', 'summary', 'content', 'type_code', 'topic_code', 'category_code', 'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on'