Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Vision ID: Standardize Site Domain and Journal objects with string-only IDs and searchable mapping

Browse Source
This commit is contained in:
Scott Idem
2026-01-19 15:57:00 -05:00
parent 2dbf47d874
commit 7db937f8af
10 changed files with 272 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
GEMINI.md
View File

@@ -56,3 +56,19 @@ This session focused on infrastructure automation and resolving critical environ
- **Storage**: Critical assets at `/home/scott/OSIT/hosted_files/` (Synced via Syncthing). - **Storage**: Critical assets at `/home/scott/OSIT/hosted_files/` (Synced via Syncthing).
- **Agents Sync**: Shared documentation and notifications pushed to `~/agents_sync/`. - **Agents Sync**: Shared documentation and notifications pushed to `~/agents_sync/`.
- **Coding Standards**: Adhering to `prompts/coding_standards.md`. - **Coding Standards**: Adhering to `prompts/coding_standards.md`.
## 📋 Aether API Development Protocol
0. Pre-Flight Check: Verify git status. Ensure all previous working changes are committed to maintain a "known good" state before starting a new cycle.
1. Strategic Plan: Write a concise plan identifying the issue, the specific files to be modified, and the verification steps (including specific curl commands or test scripts).
2. Implementation: Perform atomic code modifications using the replace or write_file tools.
3. Syntax Validation: Run python3 -m py_compile <modified_file> immediately. If syntax errors exist, they must be fixed before proceeding to deployment.
4. Process Cycle: Restart the Docker FastAPI service to apply changes:
* "docker restart aether_container_env-ae_api-2"
5. Empirical Testing:
* Execute the curl commands or test scripts identified in Step 1.
* Inspect the live log stream for regressions:
1 "tail -n 20 ~/OSIT_dev/aether_container_env/logs/ae_api/aether_api.log"
6. Finalize: Once verified, commit the changes with a descriptive message and sync relevant documentation.

48
app/lib_api_crud_v3.py
View File

@@ -126,9 +126,10 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None
""" """
Sanitizes an input payload before database insertion or update. Sanitizes an input payload before database insertion or update.
1. Resolves virtual lookup fields (`*_id_random`) into their integer database IDs. 1. Resolves ID strings to integers:
2. Removes virtual lookup fields (ending in `_id_random`) that are used for API - Handles legacy `*_id_random` fields.
convenience but do not exist in the database. - Handles Vision `*_id` fields where the value is a string (e.g., account_id: "random_str").
2. Removes virtual lookup fields (ending in `_id_random`) after resolution.
3. Removes fields explicitly marked for exclusion in the model's 3. Removes fields explicitly marked for exclusion in the model's
`fields_to_exclude_from_db` ClassVar (e.g., view-only fields). `fields_to_exclude_from_db` ClassVar (e.g., view-only fields).
4. If `ignore_extra` is True, removes all fields NOT present in the model definition. 4. If `ignore_extra` is True, removes all fields NOT present in the model definition.
@@ -140,22 +141,35 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None
from app.db_sql import redis_lookup_id_random from app.db_sql import redis_lookup_id_random
# Resolve virtual _id_random fields to integer IDs (e.g., account_id_random -> account_id) # Resolve ID strings to integers
# This must happen BEFORE we delete them.
for k, v in list(data.items()): for k, v in list(data.items()):
if k.endswith('_id_random') and k != 'id_random' and v: if not v or not isinstance(v, str):
continue
target_id_field = None
obj_type_lookup = None
# Scenario A: Legacy suffix (e.g., account_id_random: "abc")
if k.endswith('_id_random') and k != 'id_random':
target_id_field = k.replace('_id_random', '_id') target_id_field = k.replace('_id_random', '_id')
# Only resolve if the integer version is missing or null obj_type_lookup = k.replace('_id_random', '')
if not data.get(target_id_field):
obj_type_lookup = k.replace('_id_random', '') # Scenario B: Vision naming (e.g., account_id: "abc")
resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup) # We only resolve if it's a string of the correct length (random ID format)
if resolved_id: elif k.endswith('_id') and 11 <= len(v) <= 22:
data[target_id_field] = resolved_id target_id_field = k
obj_type_lookup = k.replace('_id', '')
# Filter out virtual _id_random fields (e.g., account_id_random)
keys_to_remove = [k for k in data.keys() if k.endswith('_id_random') and k != 'id_random'] if target_id_field and obj_type_lookup:
for k in keys_to_remove: # Special table mapping if needed
del data[k] if obj_type_lookup == 'address_location': obj_type_lookup = 'address'
resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup)
if resolved_id:
data[target_id_field] = resolved_id
# If we were handling Scenario A, remove the original random key
if k.endswith('_id_random'):
del data[k]
# Filter out model-specific excluded fields (e.g., view-only fields) # Filter out model-specific excluded fields (e.g., view-only fields)
if hasattr(model, 'fields_to_exclude_from_db'): if hasattr(model, 'fields_to_exclude_from_db'):

61
app/lib_sql_crud.py
View File

@@ -9,8 +9,9 @@ from sqlalchemy import text, Time
from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError
from app.log import log, logger_reset from app.log import log, logger_reset
# CRITICAL: Import the global connection state from lib_sql_core # CRITICAL: Import the core module to access current global state
from app.lib_sql_core import db, sql_connect, set_last_sql_error from app import lib_sql_core
from app.lib_sql_core import sql_connect, set_last_sql_error
# Helper for resolving random IDs # Helper for resolving random IDs
from app.lib_redis_helpers import lookup_id_random_pop from app.lib_redis_helpers import lookup_id_random_pop
@@ -50,9 +51,9 @@ def sql_insert(
log.error('SQL INSERT statement could not be created. Missing params.') log.error('SQL INSERT statement could not be created. Missing params.')
return False return False
trans = db.begin() trans = lib_sql_core.db.begin()
try: try:
result_insert = db.execute(sql_insert_stmt, data) result_insert = lib_sql_core.db.execute(sql_insert_stmt, data)
trans.commit() trans.commit()
except IntegrityError as e: except IntegrityError as e:
trans.rollback() trans.rollback()
@@ -122,16 +123,16 @@ def sql_update(
else: else:
return False return False
trans = db.begin() trans = lib_sql_core.db.begin()
try: try:
result_update = db.execute(sql_update_stmt, data) result_update = lib_sql_core.db.execute(sql_update_stmt, data)
trans.commit() trans.commit()
except OperationalError: except OperationalError:
trans.rollback() trans.rollback()
log.error('Operational error (gone away?). Retrying once...') log.error('Operational error (gone away?). Retrying once...')
sql_connect(current_db=db) sql_connect(current_db=lib_sql_core.db)
try: try:
result_update = db.execute(sql_update_stmt, data) result_update = lib_sql_core.db.execute(sql_update_stmt, data)
trans.commit() trans.commit()
except Exception as e: except Exception as e:
set_last_sql_error(e) set_last_sql_error(e)
@@ -182,9 +183,9 @@ def sql_insert_or_update(
else: else:
return False return False
trans = db.begin() trans = lib_sql_core.db.begin()
try: try:
res = db.execute(stmt, data) res = lib_sql_core.db.execute(stmt, data)
trans.commit() trans.commit()
return res.lastrowid if res.lastrowid > 0 else True return res.lastrowid if res.lastrowid > 0 else True
except Exception as e: except Exception as e:
@@ -300,38 +301,32 @@ def sql_select(
# ### END ### Core Help CRUD ### sql_select() ### # ### END ### Core Help CRUD ### sql_select() ###
# ### BEGIN ### Core Help CRUD ### run_sql_select() ### # ### BEGIN ### API DB SQL ### run_sql_select() ###
@logger_reset @logger_reset
def run_sql_select( def run_sql_select(
sql: Any, sql: text,
data: dict|None = None, data: dict|None = None,
commit: bool = False,
log_lvl: int = logging.WARNING, log_lvl: int = logging.WARNING,
) -> Any: ) -> Any:
log.setLevel(log_lvl) log.setLevel(log_lvl)
if not db:
return False print(f"Executing SQL: {sql} with data: {data}", flush=True)
try: try:
if commit: trans = db.begin() return lib_sql_core.db.execute(sql, data)
sql = sql.columns(recurring_start_time=Time, recurring_end_time=Time) except (OperationalError, ProgrammingError) as e:
result = db.execute(sql, data) if data else db.execute(sql) log.error(f'DB Error: {e}. Retrying once...')
if commit: trans.commit() sql_connect(current_db=lib_sql_core.db)
return result
except (OperationalError, ProgrammingError):
log.error('DB Error. Retrying once...')
sql_connect(current_db=db)
try: try:
if commit: trans = db.begin() return lib_sql_core.db.execute(sql, data)
result = db.execute(sql, data) if data else db.execute(sql) except Exception as e2:
if commit: trans.commit() set_last_sql_error(e2)
return result raise e2 # RAISING instead of returning False
except Exception:
return False
except Exception as e: except Exception as e:
log.exception(e) log.exception(e)
return False set_last_sql_error(e)
# ### END ### Core Help CRUD ### run_sql_select() ### raise e # RAISING instead of returning False
# ### END ### API DB SQL ### run_sql_select() ###
# ### BEGIN ### Core Help CRUD ### sql_delete() ### # ### BEGIN ### Core Help CRUD ### sql_delete() ###
@logger_reset @logger_reset
@@ -360,7 +355,7 @@ def sql_delete(
return False return False
try: try:
result = db.execute(stmt, data) if data else db.execute(stmt) result = lib_sql_core.db.execute(stmt, data) if data else lib_sql_core.db.execute(stmt)
return True if result.rowcount >= 1 else None return True if result.rowcount >= 1 else None
except Exception as e: except Exception as e:
log.exception(e) log.exception(e)

53
app/lib_sql_search.py
View File

@@ -78,12 +78,12 @@ def sql_fulltext_qry_part(fulltext_qry_dict: dict) -> tuple[str, dict]|bool:
def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool: def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool:
"""Handles enabled/disabled status filtering with schema check.""" """Handles enabled/disabled status filtering with schema check."""
from app.db_sql import db from app import lib_sql_core
if not table_name: return False if not table_name: return False
if enabled in ['enabled', 'disabled', 'all']: if enabled in ['enabled', 'disabled', 'all']:
if enabled == 'all': return '', None if enabled == 'all': return '', None
try: try:
db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0")) lib_sql_core.db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0"))
except: except:
log.warning(f"Table '{table_name}' missing 'enable' column. Skipping filter.") log.warning(f"Table '{table_name}' missing 'enable' column. Skipping filter.")
return '', None return '', None
@@ -93,12 +93,12 @@ def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool
def sql_hidden_part(table_name: str, hidden: str) -> tuple[str, bool|None]|bool: def sql_hidden_part(table_name: str, hidden: str) -> tuple[str, bool|None]|bool:
"""Handles hidden status filtering with schema check.""" """Handles hidden status filtering with schema check."""
from app.db_sql import db from app import lib_sql_core
if not table_name: return False if not table_name: return False
if hidden in ['hidden', 'not_hidden', 'all']: if hidden in ['hidden', 'not_hidden', 'all']:
if hidden == 'all': return '', None if hidden == 'all': return '', None
try: try:
db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0")) lib_sql_core.db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0"))
except: except:
log.warning(f"Table '{table_name}' missing 'hide' column. Skipping filter.") log.warning(f"Table '{table_name}' missing 'hide' column. Skipping filter.")
return '', None return '', None
@@ -133,7 +133,7 @@ def sql_search_qry_part(
table_name: str|None = None, table_name: str|None = None,
) -> tuple[str, dict]: ) -> tuple[str, dict]:
"""Recursively builds a SQL WHERE clause from a SearchQuery model.""" """Recursively builds a SQL WHERE clause from a SearchQuery model."""
from app.db_sql import db from app import lib_sql_core
data = {} data = {}
param_counter = [0] param_counter = [0]
@@ -157,9 +157,13 @@ def sql_search_qry_part(
else: else:
use_match = True use_match = True
if table_name: if table_name:
try: db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0")) try:
except: use_match = False lib_sql_core.db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0"))
else: use_match = False except:
use_match = False
else:
use_match = False
if use_match: if use_match:
p_name = get_param_name() p_name = get_param_name()
clauses.append(f"MATCH( default_qry_str ) AGAINST( :{p_name} IN BOOLEAN MODE )") clauses.append(f"MATCH( default_qry_str ) AGAINST( :{p_name} IN BOOLEAN MODE )")
@@ -172,7 +176,7 @@ def sql_search_qry_part(
'created_on', 'updated_on' 'created_on', 'updated_on'
] ]
for field in searchable_fields: for field in searchable_fields:
# Exclude exact internal integer IDs (ending in _id) # Exclude internal integer IDs specifically
if field.endswith('_id') or field == 'id': if field.endswith('_id') or field == 'id':
continue continue
@@ -183,6 +187,7 @@ def sql_search_qry_part(
f_p_name = get_param_name() f_p_name = get_param_name()
like_clauses.append(f"`{field}` LIKE :{f_p_name}") like_clauses.append(f"`{field}` LIKE :{f_p_name}")
data[f_p_name] = f"%{query_node.query_string}%" data[f_p_name] = f"%{query_node.query_string}%"
if like_clauses: clauses.append(f"({' OR '.join(like_clauses)})") if like_clauses: clauses.append(f"({' OR '.join(like_clauses)})")
for filter_attr in ['and_filters', 'or_filters']: for filter_attr in ['and_filters', 'or_filters']:
if hasattr(query_node, filter_attr) and getattr(query_node, filter_attr): if hasattr(query_node, filter_attr) and getattr(query_node, filter_attr):
@@ -198,19 +203,33 @@ def sql_search_qry_part(
return ' AND '.join(clauses) return ' AND '.join(clauses)
def process_filter(f) -> tuple[str, dict]: def process_filter(f) -> tuple[str, dict]:
if searchable_fields is not None and f.field not in searchable_fields: # --- ID VISION MAPPING ---
raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}'") # If the frontend uses clean names (id, account_id),
# map them to the database columns (id_random, account_id_random).
target_field = f.field
vision_fields = ['id', 'account_id', 'site_id', 'person_id', 'user_id', 'journal_id', 'journal_entry_id']
if target_field in vision_fields:
if target_field == 'id': target_field = 'id_random'
else: target_field = f"{target_field}_random"
print(f"Search Trace: Mapping filter field '{f.field}' -> '{target_field}'", flush=True)
if searchable_fields is not None and target_field not in searchable_fields:
# Fallback check for original field just in case
if f.field not in searchable_fields:
raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}' (mapped to '{target_field}')")
sql_op = operator_map.get(f.op.lower()) sql_op = operator_map.get(f.op.lower())
if not sql_op: raise HTTPException(status_code=400, detail=f"Unsupported operator: {f.op}") if not sql_op: raise HTTPException(status_code=400, detail=f"Unsupported operator: {f.op}")
filter_data = {} filter_data = {}
if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{f.field}` {sql_op}" if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{target_field}` {sql_op}"
else: else:
p_name = get_param_name() p_name = get_param_name()
if f.op.lower() == 'in': clause = f"`{f.field}` IN (:{p_name})"; filter_data[p_name] = f.value if f.op.lower() == 'in': clause = f"`{target_field}` IN (:{p_name})"; filter_data[p_name] = f.value
elif f.op.lower() in ['contains', 'icontains']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%" elif f.op.lower() in ['contains', 'icontains']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%"
elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%" elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%"
elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}" elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}"
else: clause = f"`{f.field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value else: clause = f"`{target_field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value
return clause, filter_data return clause, filter_data
sql_where = process_node(search_query, 1) sql_where = process_node(search_query, 1)

24
app/main.py
View File

@@ -46,6 +46,14 @@ async def lifespan(app: FastAPI):
# 2. Bootstrapping Configuration from DB with robust error handling # 2. Bootstrapping Configuration from DB with robust error handling
log.info("Bootstrapping Configuration...") log.info("Bootstrapping Configuration...")
# Save original settings for fallback
orig_db_server = config.settings.DB_SERVER
orig_db_user = config.settings.DB_USER
orig_db_pass = config.settings.DB_PASS
orig_db_name = config.settings.DB_NAME
orig_db_port = config.settings.DB_PORT
try: try:
if bootstrap_db_config(config.settings): if bootstrap_db_config(config.settings):
log.info("Successfully bootstrapped configuration from database.") log.info("Successfully bootstrapped configuration from database.")
@@ -53,13 +61,25 @@ async def lifespan(app: FastAPI):
if reconnect_db(): if reconnect_db():
log.info("Database connection re-established with production configuration.") log.info("Database connection re-established with production configuration.")
else: else:
log.warning("FAILED to re-establish database connection after bootstrap. Falling back to .env settings.") log.warning("FAILED to re-establish database connection after bootstrap. Reverting to .env settings.")
config.settings.DB_SERVER = orig_db_server
config.settings.DB_USER = orig_db_user
config.settings.DB_PASS = orig_db_pass
config.settings.DB_NAME = orig_db_name
config.settings.DB_PORT = orig_db_port
reconnect_db()
else: else:
log.warning("System bootstrap from DB returned no results. Using environment defaults.") log.warning("System bootstrap from DB returned no results. Using environment defaults.")
except Exception as e: except Exception as e:
log.error(f"Unexpected error during configuration bootstrap: {e}. Falling back to .env settings.") log.error(f"Unexpected error during configuration bootstrap: {e}. Falling back to .env settings.")
config.settings.DB_SERVER = orig_db_server
config.settings.DB_USER = orig_db_user
config.settings.DB_PASS = orig_db_pass
config.settings.DB_NAME = orig_db_name
config.settings.DB_PORT = orig_db_port
reconnect_db()
# 2. Final validation of critical infrastructure # 3. Final validation of critical infrastructure
validate_critical_config(config.settings) validate_critical_config(config.settings)
log.info('### **** *** ** * Aether API v4 using FastAPI - Startup Sequence Complete * ** *** **** ###') log.info('### **** *** ** * Aether API v4 using FastAPI - Startup Sequence Complete * ** *** **** ###')

83
app/models/journal_entry_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz import datetime, pytz
from typing import Dict, List, Optional, Set, Union, ClassVar from typing import Dict, List, Optional, Set, Union, ClassVar
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import redis_lookup_id_random from app.db_sql import redis_lookup_id_random
from app.lib_general import log, logging from app.lib_general import log, logging
@@ -14,23 +14,18 @@ class Journal_Entry_Base(BaseModel):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
id_random: Optional[str] = Field( # --- Standardized Vision IDs (Strings) ---
**base_fields['journal_entry_id_random'], id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
alias = 'journal_entry_id_random', journal_entry_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
) journal_id: Optional[str] = Field(None, **base_fields['journal_id_random'])
id: Optional[int] = Field( account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
alias = 'journal_entry_id'
)
journal_id_random: Optional[str]
journal_id: Optional[int]
external_id: Optional[str] # ID generated by or for external systems (should be stable and not change) external_id: Optional[str] # ID generated by or for external systems (should be stable and not change)
import_id: Optional[str] # Used for import purposes to track the source of the data import_id: Optional[str] # Used for import purposes to track the source of the data
code: Optional[str] code: Optional[str]
for_type: Optional[str] # 'person', 'user', 'account', etc for_type: Optional[str] # 'person', 'user', 'account', etc
for_id: Optional[int] for_id: Optional[int] = Field(None, exclude=True)
for_id_random: Optional[str] for_id_random: Optional[str]
name: Optional[str] name: Optional[str]
@@ -76,10 +71,11 @@ class Journal_Entry_Base(BaseModel):
personal: Optional[bool] = True personal: Optional[bool] = True
professional: Optional[bool] = False professional: Optional[bool] = False
parent_id_random: Optional[str] parent_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
parent_id: Optional[int] # parent_id_random: Optional[str]
related_entry_id_random: Optional[List[str]] related_entry_id_random: Optional[List[str]]
related_entry_id_li: Optional[List[int]] related_entry_id_li: Optional[List[int]] = Field(None, exclude=True)
due_datetime: Optional[datetime.datetime] due_datetime: Optional[datetime.datetime]
due_alert: Optional[bool] due_alert: Optional[bool]
@@ -113,41 +109,38 @@ class Journal_Entry_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True) @root_validator(pre=True)
def journal_entry_id_lookup(cls, v, values, **kwargs): def map_v3_ids(cls, values):
if isinstance(v, int) and v > 0: return v """
elif id_random := values.get('id_random'): Vision Transformer:
return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry') Map DB keys to clean API keys and strip internal integers.
return None """
# 1. Map Random Strings to Clean Names
@validator('journal_id', always=True) if rid := values.get('id_random') or values.get('journal_entry_id_random'):
def journal_id_lookup(cls, v, values, **kwargs): values['id'] = rid
if isinstance(v, int) and v > 0: return v values['journal_entry_id'] = rid
elif id_random := values.get('journal_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal') if j_rid := values.get('journal_id_random'):
return None values['journal_id'] = j_rid
@validator('parent_id', always=True) if a_rid := values.get('account_id_random'):
def parent_id_lookup(cls, v, values, **kwargs): values['account_id'] = a_rid
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('parent_id_random'): if p_rid := values.get('parent_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry') values['parent_id'] = p_rid
return None
# 2. Prevent "Collision Population"
@validator('for_id', always=True) for k in ['id', 'journal_id', 'account_id', 'parent_id']:
def for_id_lookup(cls, v, values, **kwargs): if k in values and not isinstance(values[k], str):
log.setLevel(logging.WARNING) del values[k]
log.debug(locals())
return values
if values.get('for_id_random') and values.get('for_type'):
return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
return None
# Fields that are part of the model (for reading) but should not be saved to the DB table # Fields that are part of the model (for reading) but should not be saved to the DB table
fields_to_exclude_from_db: ClassVar[list] = ['file_count'] fields_to_exclude_from_db: ClassVar[list] = ['file_count']
class Config: class Config:
underscore_attrs_are_private = True underscore_attrs_are_private = True
allow_population_by_field_name = True allow_population_by_field_name = False
fields = base_fields fields = base_fields
# ### END ### API Journal Entry Models ### Journal_Entry_Base() ### # ### END ### API Journal Entry Models ### Journal_Entry_Base() ###

96
app/models/journal_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz import datetime, pytz
from typing import Dict, List, Optional, Set, Union, ClassVar from typing import Dict, List, Optional, Set, Union, ClassVar
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import redis_lookup_id_random from app.db_sql import redis_lookup_id_random
from app.lib_general import log, logging from app.lib_general import log, logging
@@ -16,28 +16,19 @@ class Journal_Base(BaseModel):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
id_random: Optional[str] = Field( # --- Standardized Vision IDs (Strings) ---
**base_fields['journal_id_random'], id: Optional[str] = Field(None, **base_fields['journal_id_random'])
alias = 'journal_id_random', journal_id: Optional[str] = Field(None, **base_fields['journal_id_random'])
) account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
id: Optional[int] = Field( person_id: Optional[str] = Field(None, **base_fields['person_id_random'])
alias = 'journal_id' user_id: Optional[str] = Field(None, **base_fields['user_id_random'])
)
account_id_random: Optional[str]
account_id: Optional[int]
person_id_random: Optional[str]
person_id: Optional[int]
user_id_random: Optional[str]
user_id: Optional[int]
external_id: Optional[str] # ID generated by or for external systems (should be stable and not change) external_id: Optional[str] # ID generated by or for external systems (should be stable and not change)
import_id: Optional[str] # Used for import purposes to track the source of the data import_id: Optional[str] # Used for import purposes to track the source of the data
code: Optional[str] code: Optional[str]
for_type: Optional[str] # 'person', 'user', 'account', etc for_type: Optional[str] # 'person', 'user', 'account', etc
for_id: Optional[int] for_id: Optional[int] = Field(None, exclude=True)
for_id_random: Optional[str] for_id_random: Optional[str]
name: Optional[str] name: Optional[str]
@@ -137,51 +128,30 @@ class Journal_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True) @root_validator(pre=True)
def journal_id_lookup(cls, v, values, **kwargs): def map_v3_ids(cls, values):
if isinstance(v, int) and v > 0: return v """
elif id_random := values.get('id_random'): Vision Transformer:
return redis_lookup_id_random(record_id_random=id_random, table_name='journal') Map DB keys to clean API keys and strip internal integers.
return None """
# 1. Map Random Strings to Clean Names
@validator('account_id', always=True) if rid := values.get('id_random') or values.get('journal_id_random'):
def account_id_lookup(cls, v, values, **kwargs): values['id'] = rid
log.setLevel(logging.WARNING) values['journal_id'] = rid
log.debug(locals())
if a_rid := values.get('account_id_random'):
if isinstance(v, int) and v > 0: return v values['account_id'] = a_rid
elif id_random := values.get('account_id_random'): if p_rid := values.get('person_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='account') values['person_id'] = p_rid
return None if u_rid := values.get('user_id_random'):
values['user_id'] = u_rid
@validator('person_id', always=True)
def person_id_lookup(cls, v, values, **kwargs): # 2. Prevent "Collision Population"
log.setLevel(logging.WARNING) for k in ['id', 'account_id', 'person_id', 'user_id']:
log.debug(locals()) if k in values and not isinstance(values[k], str):
del values[k]
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('person_id_random'): return values
return redis_lookup_id_random(record_id_random=id_random, table_name='person')
return None
@validator('user_id', always=True)
def user_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('user_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='user')
return None
@validator('for_id', always=True)
def for_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values.get('for_id_random') and values.get('for_type'):
return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
return None
# Fields that are part of the model (for reading) but should not be saved to the DB table # Fields that are part of the model (for reading) but should not be saved to the DB table
fields_to_exclude_from_db: ClassVar[list] = [ fields_to_exclude_from_db: ClassVar[list] = [
@@ -192,6 +162,6 @@ class Journal_Base(BaseModel):
class Config: class Config:
underscore_attrs_are_private = True underscore_attrs_are_private = True
allow_population_by_field_name = True allow_population_by_field_name = False
fields = base_fields fields = base_fields
# ### END ### API Journal Models ### Journal_Base() ### # ### END ### API Journal Models ### Journal_Base() ###

183
app/models/site_domain_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz import datetime, pytz
from typing import Dict, List, Optional, Set, Union from typing import Dict, List, Optional, Set, Union
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import get_id_random, redis_lookup_id_random from app.db_sql import get_id_random, redis_lookup_id_random
from app.lib_general import log, logging from app.lib_general import log, logging
@@ -14,16 +14,11 @@ class Site_Domain_Base(BaseModel):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
id_random: Optional[str] = Field( # --- Standardized Vision IDs (Strings) ---
**base_fields['site_domain_id_random'], id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
alias = 'site_domain_id_random', site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
) site_id: Optional[str] = Field(None, **base_fields['site_id_random'])
id: Optional[int] = Field( account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
alias = 'site_domain_id'
)
site_id_random: Optional[str]
site_id: Optional[int]
fqdn: Optional[str] fqdn: Optional[str]
@@ -38,59 +33,45 @@ class Site_Domain_Base(BaseModel):
cfg_json: Optional[Union[Json, None]] # In use 2024-03-04 cfg_json: Optional[Union[Json, None]] # In use 2024-03-04
hide: Optional[bool] hide: Optional[bool] = None # Field missing in physical table but common in views
priority: Optional[bool] # priority: Optional[bool] # MISSING in physical table
sort: Optional[int] # sort: Optional[int] # MISSING in physical table
group: Optional[str] # group: Optional[str] # MISSING in physical table
notes: Optional[str] notes: Optional[str] = None # MISSING in physical table
created_on: Optional[datetime.datetime] = None created_on: Optional[datetime.datetime] = None
updated_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None
# Including convenience data
# This is only for convenience. Probably going to keep unless it causes a problem.
account_id: Optional[int]
account_id_random: Optional[str]
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True) @root_validator(pre=True)
def site_domain_id_lookup(cls, v, values, **kwargs): def map_v3_ids(cls, values):
log.setLevel(logging.WARNING) """
log.debug(locals()) Vision Transformer:
Map DB-centric keys to clean API keys and strip internal integers.
if isinstance(v, int) and v > 0: return v """
elif id_random := values.get('id_random'): # 1. Map Random Strings to Clean Names
return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain') # We prioritize the random strings to ensure the Vision is string-based.
return None if rid := values.get('id_random') or values.get('site_domain_id_random'):
values['id'] = rid
@validator('account_id', always=True) values['site_domain_id'] = rid
def account_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v if s_rid := values.get('site_id_random'):
elif id_random := values.get('account_id_random'): values['site_id'] = s_rid
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
return None if a_rid := values.get('account_id_random'):
values['account_id'] = a_rid
@validator('account_id_random', always=True)
def account_id_random_lookup(cls, v, values, **kwargs): # 2. Prevent "Collision Population"
if isinstance(v, str) and len(v) >= 11: return v for k in ['id', 'site_id', 'account_id']:
elif account_id := values.get('account_id'): if k in values and not isinstance(values[k], str):
return get_id_random(record_id=account_id, table_name='account') del values[k]
return None
return values
@validator('site_id', always=True)
def site_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_id_random']:
return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site')
return None
class Config: class Config:
underscore_attrs_are_private = True underscore_attrs_are_private = True
fields = base_fields allow_population_by_field_name = False
allow_population_by_field_name = True
# ### END ### API Site Domain Models ### Site_Domain_Base() ### # ### END ### API Site Domain Models ### Site_Domain_Base() ###
@@ -99,19 +80,11 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
id_random: Optional[str] = Field( # --- Standardized Vision IDs (Strings) ---
**base_fields['site_domain_id_random'], id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
alias = 'site_domain_id_random', site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
) site_id: Optional[str] = Field(None, **base_fields['site_id_random'])
id: Optional[int] = Field( account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
alias = 'site_domain_id'
)
site_id_random: Optional[str]
site_id: Optional[int]
site_domain_id_random: Optional[str]
site_domain_id: Optional[int]
fqdn: Optional[str] fqdn: Optional[str]
@@ -124,19 +97,13 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
valid_for: Optional[int] # number of hours valid_for: Optional[int] # number of hours
enable: Optional[bool] enable: Optional[bool]
hide: Optional[bool] hide: Optional[bool] = None
priority: Optional[bool]
sort: Optional[int] notes: Optional[str] = None
group: Optional[str]
notes: Optional[str]
created_on: Optional[datetime.datetime] = None created_on: Optional[datetime.datetime] = None
updated_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None
# Including convenience data # Including convenience data
# This is only for convenience. Probably going to keep unless it causes a problem.
account_id: Optional[int]
account_id_random: Optional[str]
account_code: Optional[str] # Useful for export file naming account_code: Optional[str] # Useful for export file naming
account_name: Optional[str] # Generally useful for display account_name: Optional[str] # Generally useful for display
account_enable: Optional[bool] account_enable: Optional[bool]
@@ -157,54 +124,22 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True) @root_validator(pre=True)
def id_lookup(cls, v, values, **kwargs): def map_v3_ids(cls, values):
if isinstance(v, int) and v > 0: return v if rid := values.get('id_random') or values.get('site_domain_id_random'):
elif id_random := values.get('id_random'): values['id'] = rid
return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain') values['site_domain_id'] = rid
return None if s_rid := values.get('site_id_random'):
values['site_id'] = s_rid
@validator('site_domain_id_random', always=True) if a_rid := values.get('account_id_random'):
def site_domain_id_random_lookup(cls, v, values, **kwargs): values['account_id'] = a_rid
if isinstance(v, str) and len(v) >= 11: return v
elif id_random := values.get('id_random'): for k in ['id', 'site_id', 'account_id']:
return id_random if k in values and not isinstance(values[k], str):
return None del values[k]
return values
@validator('account_id', always=True)
def account_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('account_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
return None
@validator('account_id_random', always=True)
def account_id_random_lookup(cls, v, values, **kwargs):
if isinstance(v, str) and len(v) >= 11: return v
elif account_id := values.get('account_id'):
return get_id_random(record_id=account_id, table_name='account')
return None
@validator('site_id', always=True)
def site_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_id_random']:
return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site')
return None
@validator('site_domain_id', always=True)
def site_domain_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_domain_id_random']:
return redis_lookup_id_random(record_id_random=values['site_domain_id_random'], table_name='site')
return None
class Config: class Config:
underscore_attrs_are_private = True underscore_attrs_are_private = True
fields = base_fields allow_population_by_field_name = False
allow_population_by_field_name = True
# ### END ### API Site Domain Models ### Site_Domain_FQDN_ID_Base() ### # ### END ### API Site Domain Models ### Site_Domain_FQDN_ID_Base() ###

5
app/object_definitions/cms.py
View File

@@ -116,8 +116,9 @@ cms_obj_li = {
'base_name_alt': Site_Domain_FQDN_ID_Base, 'base_name_alt': Site_Domain_FQDN_ID_Base,
# V3 Search Security: # V3 Search Security:
'searchable_fields': [ 'searchable_fields': [
'site_domain_id_random', 'account_id_random', 'site_id_random', 'id', 'account_id', 'site_id',
'fqdn', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on' 'id_random', 'account_id_random', 'site_id_random',
'fqdn', 'enable', 'created_on', 'updated_on'
], ],
}, },
} }

8
app/object_definitions/journals.py
View File

@@ -20,7 +20,9 @@ journal_obj_li = {
], ],
# V3 Search Security: # V3 Search Security:
'searchable_fields': [ 'searchable_fields': [
'journal_id_random', 'name', 'short_name', 'summary', 'outline', 'id', 'account_id', 'person_id', 'user_id',
'journal_id_random', 'account_id_random', 'person_id_random', 'user_id_random',
'name', 'short_name', 'summary', 'outline',
'description', 'type_code', 'tags', 'billable', 'enable', 'hide', 'description', 'type_code', 'tags', 'billable', 'enable', 'hide',
'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on' 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on'
], ],
@@ -42,7 +44,9 @@ journal_obj_li = {
], ],
# V3 Search Security: # V3 Search Security:
'searchable_fields': [ 'searchable_fields': [
'journal_entry_id_random', 'journal_id_random', 'name', 'short_name', 'id', 'journal_id', 'account_id',
'journal_entry_id_random', 'journal_id_random', 'account_id_random',
'name', 'short_name',
'summary', 'content', 'type_code', 'topic_code', 'category_code', 'summary', 'content', 'type_code', 'topic_code', 'category_code',
'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'sort', 'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'sort',
'group', 'notes', 'created_on', 'updated_on' 'group', 'notes', 'created_on', 'updated_on'