Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Vision ID: Standardize Site Domain and Journal objects with string-only IDs and searchable mapping

Browse Source
This commit is contained in:
Scott Idem
2026-01-19 15:57:00 -05:00
parent 2dbf47d874
commit 7db937f8af
10 changed files with 272 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
GEMINI.md
View File

@@ -56,3 +56,19 @@ This session focused on infrastructure automation and resolving critical environ
- **Storage**: Critical assets at `/home/scott/OSIT/hosted_files/` (Synced via Syncthing).
- **Agents Sync**: Shared documentation and notifications pushed to `~/agents_sync/`.
- **Coding Standards**: Adhering to `prompts/coding_standards.md`.
## 📋 Aether API Development Protocol
0. Pre-Flight Check: Verify git status. Ensure all previous working changes are committed to maintain a "known good" state before starting a new cycle.
1. Strategic Plan: Write a concise plan identifying the issue, the specific files to be modified, and the verification steps (including specific curl commands or test scripts).
2. Implementation: Perform atomic code modifications using the replace or write_file tools.
3. Syntax Validation: Run python3 -m py_compile <modified_file> immediately. If syntax errors exist, they must be fixed before proceeding to deployment.
4. Process Cycle: Restart the Docker FastAPI service to apply changes:
* "docker restart aether_container_env-ae_api-2"
5. Empirical Testing:
* Execute the curl commands or test scripts identified in Step 1.
* Inspect the live log stream for regressions:
1 "tail -n 20 ~/OSIT_dev/aether_container_env/logs/ae_api/aether_api.log"
6. Finalize: Once verified, commit the changes with a descriptive message and sync relevant documentation.

48
app/lib_api_crud_v3.py
View File

@@ -126,9 +126,10 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None
"""
Sanitizes an input payload before database insertion or update.
1. Resolves virtual lookup fields (`*_id_random`) into their integer database IDs.
2. Removes virtual lookup fields (ending in `_id_random`) that are used for API
convenience but do not exist in the database.
1. Resolves ID strings to integers:
- Handles legacy `*_id_random` fields.
- Handles Vision `*_id` fields where the value is a string (e.g., account_id: "random_str").
2. Removes virtual lookup fields (ending in `_id_random`) after resolution.
3. Removes fields explicitly marked for exclusion in the model's
`fields_to_exclude_from_db` ClassVar (e.g., view-only fields).
4. If `ignore_extra` is True, removes all fields NOT present in the model definition.
@@ -140,22 +141,35 @@ def sanitize_payload(data: dict, model: Any, ignore_extra: bool = False) -> None
from app.db_sql import redis_lookup_id_random
# Resolve virtual _id_random fields to integer IDs (e.g., account_id_random -> account_id)
# This must happen BEFORE we delete them.
# Resolve ID strings to integers
for k, v in list(data.items()):
if k.endswith('_id_random') and k != 'id_random' and v:
if not v or not isinstance(v, str):
continue
target_id_field = None
obj_type_lookup = None
# Scenario A: Legacy suffix (e.g., account_id_random: "abc")
if k.endswith('_id_random') and k != 'id_random':
target_id_field = k.replace('_id_random', '_id')
# Only resolve if the integer version is missing or null
if not data.get(target_id_field):
obj_type_lookup = k.replace('_id_random', '')
resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup)
if resolved_id:
data[target_id_field] = resolved_id
# Filter out virtual _id_random fields (e.g., account_id_random)
keys_to_remove = [k for k in data.keys() if k.endswith('_id_random') and k != 'id_random']
for k in keys_to_remove:
del data[k]
obj_type_lookup = k.replace('_id_random', '')
# Scenario B: Vision naming (e.g., account_id: "abc")
# We only resolve if it's a string of the correct length (random ID format)
elif k.endswith('_id') and 11 <= len(v) <= 22:
target_id_field = k
obj_type_lookup = k.replace('_id', '')
if target_id_field and obj_type_lookup:
# Special table mapping if needed
if obj_type_lookup == 'address_location': obj_type_lookup = 'address'
resolved_id = redis_lookup_id_random(record_id_random=v, table_name=obj_type_lookup)
if resolved_id:
data[target_id_field] = resolved_id
# If we were handling Scenario A, remove the original random key
if k.endswith('_id_random'):
del data[k]
# Filter out model-specific excluded fields (e.g., view-only fields)
if hasattr(model, 'fields_to_exclude_from_db'):

61
app/lib_sql_crud.py
View File

@@ -9,8 +9,9 @@ from sqlalchemy import text, Time
from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError
from app.log import log, logger_reset
# CRITICAL: Import the global connection state from lib_sql_core
from app.lib_sql_core import db, sql_connect, set_last_sql_error
# CRITICAL: Import the core module to access current global state
from app import lib_sql_core
from app.lib_sql_core import sql_connect, set_last_sql_error
# Helper for resolving random IDs
from app.lib_redis_helpers import lookup_id_random_pop
@@ -50,9 +51,9 @@ def sql_insert(
log.error('SQL INSERT statement could not be created. Missing params.')
return False
trans = db.begin()
trans = lib_sql_core.db.begin()
try:
result_insert = db.execute(sql_insert_stmt, data)
result_insert = lib_sql_core.db.execute(sql_insert_stmt, data)
trans.commit()
except IntegrityError as e:
trans.rollback()
@@ -122,16 +123,16 @@ def sql_update(
else:
return False
trans = db.begin()
trans = lib_sql_core.db.begin()
try:
result_update = db.execute(sql_update_stmt, data)
result_update = lib_sql_core.db.execute(sql_update_stmt, data)
trans.commit()
except OperationalError:
trans.rollback()
log.error('Operational error (gone away?). Retrying once...')
sql_connect(current_db=db)
sql_connect(current_db=lib_sql_core.db)
try:
result_update = db.execute(sql_update_stmt, data)
result_update = lib_sql_core.db.execute(sql_update_stmt, data)
trans.commit()
except Exception as e:
set_last_sql_error(e)
@@ -182,9 +183,9 @@ def sql_insert_or_update(
else:
return False
trans = db.begin()
trans = lib_sql_core.db.begin()
try:
res = db.execute(stmt, data)
res = lib_sql_core.db.execute(stmt, data)
trans.commit()
return res.lastrowid if res.lastrowid > 0 else True
except Exception as e:
@@ -300,38 +301,32 @@ def sql_select(
# ### END ### Core Help CRUD ### sql_select() ###
# ### BEGIN ### Core Help CRUD ### run_sql_select() ###
# ### BEGIN ### API DB SQL ### run_sql_select() ###
@logger_reset
def run_sql_select(
sql: Any,
sql: text,
data: dict|None = None,
commit: bool = False,
log_lvl: int = logging.WARNING,
) -> Any:
log.setLevel(log_lvl)
if not db:
return False
print(f"Executing SQL: {sql} with data: {data}", flush=True)
try:
if commit: trans = db.begin()
sql = sql.columns(recurring_start_time=Time, recurring_end_time=Time)
result = db.execute(sql, data) if data else db.execute(sql)
if commit: trans.commit()
return result
except (OperationalError, ProgrammingError):
log.error('DB Error. Retrying once...')
sql_connect(current_db=db)
return lib_sql_core.db.execute(sql, data)
except (OperationalError, ProgrammingError) as e:
log.error(f'DB Error: {e}. Retrying once...')
sql_connect(current_db=lib_sql_core.db)
try:
if commit: trans = db.begin()
result = db.execute(sql, data) if data else db.execute(sql)
if commit: trans.commit()
return result
except Exception:
return False
return lib_sql_core.db.execute(sql, data)
except Exception as e2:
set_last_sql_error(e2)
raise e2 # RAISING instead of returning False
except Exception as e:
log.exception(e)
return False
# ### END ### Core Help CRUD ### run_sql_select() ###
set_last_sql_error(e)
raise e # RAISING instead of returning False
# ### END ### API DB SQL ### run_sql_select() ###
# ### BEGIN ### Core Help CRUD ### sql_delete() ###
@logger_reset
@@ -360,7 +355,7 @@ def sql_delete(
return False
try:
result = db.execute(stmt, data) if data else db.execute(stmt)
result = lib_sql_core.db.execute(stmt, data) if data else lib_sql_core.db.execute(stmt)
return True if result.rowcount >= 1 else None
except Exception as e:
log.exception(e)

53
app/lib_sql_search.py
View File

@@ -78,12 +78,12 @@ def sql_fulltext_qry_part(fulltext_qry_dict: dict) -> tuple[str, dict]|bool:
def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool:
"""Handles enabled/disabled status filtering with schema check."""
from app.db_sql import db
from app import lib_sql_core
if not table_name: return False
if enabled in ['enabled', 'disabled', 'all']:
if enabled == 'all': return '', None
try:
db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0"))
lib_sql_core.db.execute(text(f"SELECT enable FROM `{table_name}` LIMIT 0"))
except:
log.warning(f"Table '{table_name}' missing 'enable' column. Skipping filter.")
return '', None
@@ -93,12 +93,12 @@ def sql_enable_part(table_name: str, enabled: str) -> tuple[str, bool|None]|bool
def sql_hidden_part(table_name: str, hidden: str) -> tuple[str, bool|None]|bool:
"""Handles hidden status filtering with schema check."""
from app.db_sql import db
from app import lib_sql_core
if not table_name: return False
if hidden in ['hidden', 'not_hidden', 'all']:
if hidden == 'all': return '', None
try:
db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0"))
lib_sql_core.db.execute(text(f"SELECT hide FROM `{table_name}` LIMIT 0"))
except:
log.warning(f"Table '{table_name}' missing 'hide' column. Skipping filter.")
return '', None
@@ -133,7 +133,7 @@ def sql_search_qry_part(
table_name: str|None = None,
) -> tuple[str, dict]:
"""Recursively builds a SQL WHERE clause from a SearchQuery model."""
from app.db_sql import db
from app import lib_sql_core
data = {}
param_counter = [0]
@@ -157,9 +157,13 @@ def sql_search_qry_part(
else:
use_match = True
if table_name:
try: db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0"))
except: use_match = False
else: use_match = False
try:
lib_sql_core.db.execute(text(f"SELECT default_qry_str FROM `{table_name}` LIMIT 0"))
except:
use_match = False
else:
use_match = False
if use_match:
p_name = get_param_name()
clauses.append(f"MATCH( default_qry_str ) AGAINST( :{p_name} IN BOOLEAN MODE )")
@@ -172,7 +176,7 @@ def sql_search_qry_part(
'created_on', 'updated_on'
]
for field in searchable_fields:
# Exclude exact internal integer IDs (ending in _id)
# Exclude internal integer IDs specifically
if field.endswith('_id') or field == 'id':
continue
@@ -183,6 +187,7 @@ def sql_search_qry_part(
f_p_name = get_param_name()
like_clauses.append(f"`{field}` LIKE :{f_p_name}")
data[f_p_name] = f"%{query_node.query_string}%"
if like_clauses: clauses.append(f"({' OR '.join(like_clauses)})")
for filter_attr in ['and_filters', 'or_filters']:
if hasattr(query_node, filter_attr) and getattr(query_node, filter_attr):
@@ -198,19 +203,33 @@ def sql_search_qry_part(
return ' AND '.join(clauses)
def process_filter(f) -> tuple[str, dict]:
if searchable_fields is not None and f.field not in searchable_fields:
raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}'")
# --- ID VISION MAPPING ---
# If the frontend uses clean names (id, account_id),
# map them to the database columns (id_random, account_id_random).
target_field = f.field
vision_fields = ['id', 'account_id', 'site_id', 'person_id', 'user_id', 'journal_id', 'journal_entry_id']
if target_field in vision_fields:
if target_field == 'id': target_field = 'id_random'
else: target_field = f"{target_field}_random"
print(f"Search Trace: Mapping filter field '{f.field}' -> '{target_field}'", flush=True)
if searchable_fields is not None and target_field not in searchable_fields:
# Fallback check for original field just in case
if f.field not in searchable_fields:
raise HTTPException(status_code=400, detail=f"Unauthorized search field '{f.field}' (mapped to '{target_field}')")
sql_op = operator_map.get(f.op.lower())
if not sql_op: raise HTTPException(status_code=400, detail=f"Unsupported operator: {f.op}")
filter_data = {}
if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{f.field}` {sql_op}"
if f.op.lower() in ['is_null', 'is_not_null']: clause = f"`{target_field}` {sql_op}"
else:
p_name = get_param_name()
if f.op.lower() == 'in': clause = f"`{f.field}` IN (:{p_name})"; filter_data[p_name] = f.value
elif f.op.lower() in ['contains', 'icontains']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%"
elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%"
elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{f.field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}"
else: clause = f"`{f.field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value
if f.op.lower() == 'in': clause = f"`{target_field}` IN (:{p_name})"; filter_data[p_name] = f.value
elif f.op.lower() in ['contains', 'icontains']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}%"
elif f.op.lower() in ['startswith', 'istartswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"{f.value}%"
elif f.op.lower() in ['endswith', 'iendswith']: clause = f"`{target_field}` LIKE :{p_name}"; filter_data[p_name] = f"%{f.value}"
else: clause = f"`{target_field}` {sql_op} :{p_name}"; filter_data[p_name] = f.value
return clause, filter_data
sql_where = process_node(search_query, 1)

24
app/main.py
View File

@@ -46,6 +46,14 @@ async def lifespan(app: FastAPI):
# 2. Bootstrapping Configuration from DB with robust error handling
log.info("Bootstrapping Configuration...")
# Save original settings for fallback
orig_db_server = config.settings.DB_SERVER
orig_db_user = config.settings.DB_USER
orig_db_pass = config.settings.DB_PASS
orig_db_name = config.settings.DB_NAME
orig_db_port = config.settings.DB_PORT
try:
if bootstrap_db_config(config.settings):
log.info("Successfully bootstrapped configuration from database.")
@@ -53,13 +61,25 @@ async def lifespan(app: FastAPI):
if reconnect_db():
log.info("Database connection re-established with production configuration.")
else:
log.warning("FAILED to re-establish database connection after bootstrap. Falling back to .env settings.")
log.warning("FAILED to re-establish database connection after bootstrap. Reverting to .env settings.")
config.settings.DB_SERVER = orig_db_server
config.settings.DB_USER = orig_db_user
config.settings.DB_PASS = orig_db_pass
config.settings.DB_NAME = orig_db_name
config.settings.DB_PORT = orig_db_port
reconnect_db()
else:
log.warning("System bootstrap from DB returned no results. Using environment defaults.")
except Exception as e:
log.error(f"Unexpected error during configuration bootstrap: {e}. Falling back to .env settings.")
config.settings.DB_SERVER = orig_db_server
config.settings.DB_USER = orig_db_user
config.settings.DB_PASS = orig_db_pass
config.settings.DB_NAME = orig_db_name
config.settings.DB_PORT = orig_db_port
reconnect_db()
# 2. Final validation of critical infrastructure
# 3. Final validation of critical infrastructure
validate_critical_config(config.settings)
log.info('### **** *** ** * Aether API v4 using FastAPI - Startup Sequence Complete * ** *** **** ###')

83
app/models/journal_entry_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz
from typing import Dict, List, Optional, Set, Union, ClassVar
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import redis_lookup_id_random
from app.lib_general import log, logging
@@ -14,23 +14,18 @@ class Journal_Entry_Base(BaseModel):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
id_random: Optional[str] = Field(
**base_fields['journal_entry_id_random'],
alias = 'journal_entry_id_random',
)
id: Optional[int] = Field(
alias = 'journal_entry_id'
)
journal_id_random: Optional[str]
journal_id: Optional[int]
# --- Standardized Vision IDs (Strings) ---
id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
journal_entry_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
journal_id: Optional[str] = Field(None, **base_fields['journal_id_random'])
account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
external_id: Optional[str] # ID generated by or for external systems (should be stable and not change)
import_id: Optional[str] # Used for import purposes to track the source of the data
code: Optional[str]
for_type: Optional[str] # 'person', 'user', 'account', etc
for_id: Optional[int]
for_id: Optional[int] = Field(None, exclude=True)
for_id_random: Optional[str]
name: Optional[str]
@@ -76,10 +71,11 @@ class Journal_Entry_Base(BaseModel):
personal: Optional[bool] = True
professional: Optional[bool] = False
parent_id_random: Optional[str]
parent_id: Optional[int]
parent_id: Optional[str] = Field(None, **base_fields['journal_entry_id_random'])
# parent_id_random: Optional[str]
related_entry_id_random: Optional[List[str]]
related_entry_id_li: Optional[List[int]]
related_entry_id_li: Optional[List[int]] = Field(None, exclude=True)
due_datetime: Optional[datetime.datetime]
due_alert: Optional[bool]
@@ -113,41 +109,38 @@ class Journal_Entry_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True)
def journal_entry_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry')
return None
@validator('journal_id', always=True)
def journal_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('journal_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal')
return None
@validator('parent_id', always=True)
def parent_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('parent_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal_entry')
return None
@validator('for_id', always=True)
def for_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values.get('for_id_random') and values.get('for_type'):
return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
return None
@root_validator(pre=True)
def map_v3_ids(cls, values):
"""
Vision Transformer:
Map DB keys to clean API keys and strip internal integers.
"""
# 1. Map Random Strings to Clean Names
if rid := values.get('id_random') or values.get('journal_entry_id_random'):
values['id'] = rid
values['journal_entry_id'] = rid
if j_rid := values.get('journal_id_random'):
values['journal_id'] = j_rid
if a_rid := values.get('account_id_random'):
values['account_id'] = a_rid
if p_rid := values.get('parent_id_random'):
values['parent_id'] = p_rid
# 2. Prevent "Collision Population"
for k in ['id', 'journal_id', 'account_id', 'parent_id']:
if k in values and not isinstance(values[k], str):
del values[k]
return values
# Fields that are part of the model (for reading) but should not be saved to the DB table
fields_to_exclude_from_db: ClassVar[list] = ['file_count']
class Config:
underscore_attrs_are_private = True
allow_population_by_field_name = True
allow_population_by_field_name = False
fields = base_fields
# ### END ### API Journal Entry Models ### Journal_Entry_Base() ###

96
app/models/journal_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz
from typing import Dict, List, Optional, Set, Union, ClassVar
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import redis_lookup_id_random
from app.lib_general import log, logging
@@ -16,28 +16,19 @@ class Journal_Base(BaseModel):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
id_random: Optional[str] = Field(
**base_fields['journal_id_random'],
alias = 'journal_id_random',
)
id: Optional[int] = Field(
alias = 'journal_id'
)
account_id_random: Optional[str]
account_id: Optional[int]
person_id_random: Optional[str]
person_id: Optional[int]
user_id_random: Optional[str]
user_id: Optional[int]
# --- Standardized Vision IDs (Strings) ---
id: Optional[str] = Field(None, **base_fields['journal_id_random'])
journal_id: Optional[str] = Field(None, **base_fields['journal_id_random'])
account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
person_id: Optional[str] = Field(None, **base_fields['person_id_random'])
user_id: Optional[str] = Field(None, **base_fields['user_id_random'])
external_id: Optional[str] # ID generated by or for external systems (should be stable and not change)
import_id: Optional[str] # Used for import purposes to track the source of the data
code: Optional[str]
for_type: Optional[str] # 'person', 'user', 'account', etc
for_id: Optional[int]
for_id: Optional[int] = Field(None, exclude=True)
for_id_random: Optional[str]
name: Optional[str]
@@ -137,51 +128,30 @@ class Journal_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True)
def journal_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='journal')
return None
@validator('account_id', always=True)
def account_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('account_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
return None
@validator('person_id', always=True)
def person_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('person_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='person')
return None
@validator('user_id', always=True)
def user_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('user_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='user')
return None
@validator('for_id', always=True)
def for_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values.get('for_id_random') and values.get('for_type'):
return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
return None
@root_validator(pre=True)
def map_v3_ids(cls, values):
"""
Vision Transformer:
Map DB keys to clean API keys and strip internal integers.
"""
# 1. Map Random Strings to Clean Names
if rid := values.get('id_random') or values.get('journal_id_random'):
values['id'] = rid
values['journal_id'] = rid
if a_rid := values.get('account_id_random'):
values['account_id'] = a_rid
if p_rid := values.get('person_id_random'):
values['person_id'] = p_rid
if u_rid := values.get('user_id_random'):
values['user_id'] = u_rid
# 2. Prevent "Collision Population"
for k in ['id', 'account_id', 'person_id', 'user_id']:
if k in values and not isinstance(values[k], str):
del values[k]
return values
# Fields that are part of the model (for reading) but should not be saved to the DB table
fields_to_exclude_from_db: ClassVar[list] = [
@@ -192,6 +162,6 @@ class Journal_Base(BaseModel):
class Config:
underscore_attrs_are_private = True
allow_population_by_field_name = True
allow_population_by_field_name = False
fields = base_fields
# ### END ### API Journal Models ### Journal_Base() ###

183
app/models/site_domain_models.py
View File

@@ -1,7 +1,7 @@
import datetime, pytz
from typing import Dict, List, Optional, Set, Union
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator
from app.db_sql import get_id_random, redis_lookup_id_random
from app.lib_general import log, logging
@@ -14,16 +14,11 @@ class Site_Domain_Base(BaseModel):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
id_random: Optional[str] = Field(
**base_fields['site_domain_id_random'],
alias = 'site_domain_id_random',
)
id: Optional[int] = Field(
alias = 'site_domain_id'
)
site_id_random: Optional[str]
site_id: Optional[int]
# --- Standardized Vision IDs (Strings) ---
id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
site_id: Optional[str] = Field(None, **base_fields['site_id_random'])
account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
fqdn: Optional[str]
@@ -38,59 +33,45 @@ class Site_Domain_Base(BaseModel):
cfg_json: Optional[Union[Json, None]] # In use 2024-03-04
hide: Optional[bool]
priority: Optional[bool]
sort: Optional[int]
group: Optional[str]
hide: Optional[bool] = None # Field missing in physical table but common in views
# priority: Optional[bool] # MISSING in physical table
# sort: Optional[int] # MISSING in physical table
# group: Optional[str] # MISSING in physical table
notes: Optional[str]
notes: Optional[str] = None # MISSING in physical table
created_on: Optional[datetime.datetime] = None
updated_on: Optional[datetime.datetime] = None
# Including convenience data
# This is only for convenience. Probably going to keep unless it causes a problem.
account_id: Optional[int]
account_id_random: Optional[str]
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True)
def site_domain_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain')
return None
@validator('account_id', always=True)
def account_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('account_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
return None
@validator('account_id_random', always=True)
def account_id_random_lookup(cls, v, values, **kwargs):
if isinstance(v, str) and len(v) >= 11: return v
elif account_id := values.get('account_id'):
return get_id_random(record_id=account_id, table_name='account')
return None
@validator('site_id', always=True)
def site_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_id_random']:
return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site')
return None
@root_validator(pre=True)
def map_v3_ids(cls, values):
"""
Vision Transformer:
Map DB-centric keys to clean API keys and strip internal integers.
"""
# 1. Map Random Strings to Clean Names
# We prioritize the random strings to ensure the Vision is string-based.
if rid := values.get('id_random') or values.get('site_domain_id_random'):
values['id'] = rid
values['site_domain_id'] = rid
if s_rid := values.get('site_id_random'):
values['site_id'] = s_rid
if a_rid := values.get('account_id_random'):
values['account_id'] = a_rid
# 2. Prevent "Collision Population"
for k in ['id', 'site_id', 'account_id']:
if k in values and not isinstance(values[k], str):
del values[k]
return values
class Config:
underscore_attrs_are_private = True
fields = base_fields
allow_population_by_field_name = True
allow_population_by_field_name = False
# ### END ### API Site Domain Models ### Site_Domain_Base() ###
@@ -99,19 +80,11 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
id_random: Optional[str] = Field(
**base_fields['site_domain_id_random'],
alias = 'site_domain_id_random',
)
id: Optional[int] = Field(
alias = 'site_domain_id'
)
site_id_random: Optional[str]
site_id: Optional[int]
site_domain_id_random: Optional[str]
site_domain_id: Optional[int]
# --- Standardized Vision IDs (Strings) ---
id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
site_domain_id: Optional[str] = Field(None, **base_fields['site_domain_id_random'])
site_id: Optional[str] = Field(None, **base_fields['site_id_random'])
account_id: Optional[str] = Field(None, **base_fields['account_id_random'])
fqdn: Optional[str]
@@ -124,19 +97,13 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
valid_for: Optional[int] # number of hours
enable: Optional[bool]
hide: Optional[bool]
priority: Optional[bool]
sort: Optional[int]
group: Optional[str]
notes: Optional[str]
hide: Optional[bool] = None
notes: Optional[str] = None
created_on: Optional[datetime.datetime] = None
updated_on: Optional[datetime.datetime] = None
# Including convenience data
# This is only for convenience. Probably going to keep unless it causes a problem.
account_id: Optional[int]
account_id_random: Optional[str]
account_code: Optional[str] # Useful for export file naming
account_name: Optional[str] # Generally useful for display
account_enable: Optional[bool]
@@ -157,54 +124,22 @@ class Site_Domain_FQDN_ID_Base(BaseModel):
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
@validator('id', always=True)
def id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='site_domain')
return None
@validator('site_domain_id_random', always=True)
def site_domain_id_random_lookup(cls, v, values, **kwargs):
if isinstance(v, str) and len(v) >= 11: return v
elif id_random := values.get('id_random'):
return id_random
return None
@validator('account_id', always=True)
def account_id_lookup(cls, v, values, **kwargs):
if isinstance(v, int) and v > 0: return v
elif id_random := values.get('account_id_random'):
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
return None
@validator('account_id_random', always=True)
def account_id_random_lookup(cls, v, values, **kwargs):
if isinstance(v, str) and len(v) >= 11: return v
elif account_id := values.get('account_id'):
return get_id_random(record_id=account_id, table_name='account')
return None
@validator('site_id', always=True)
def site_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_id_random']:
return redis_lookup_id_random(record_id_random=values['site_id_random'], table_name='site')
return None
@validator('site_domain_id', always=True)
def site_domain_id_lookup(cls, v, values, **kwargs):
log.setLevel(logging.WARNING)
log.debug(locals())
if values['site_domain_id_random']:
return redis_lookup_id_random(record_id_random=values['site_domain_id_random'], table_name='site')
return None
@root_validator(pre=True)
def map_v3_ids(cls, values):
if rid := values.get('id_random') or values.get('site_domain_id_random'):
values['id'] = rid
values['site_domain_id'] = rid
if s_rid := values.get('site_id_random'):
values['site_id'] = s_rid
if a_rid := values.get('account_id_random'):
values['account_id'] = a_rid
for k in ['id', 'site_id', 'account_id']:
if k in values and not isinstance(values[k], str):
del values[k]
return values
class Config:
underscore_attrs_are_private = True
fields = base_fields
allow_population_by_field_name = True
allow_population_by_field_name = False
# ### END ### API Site Domain Models ### Site_Domain_FQDN_ID_Base() ###

5
app/object_definitions/cms.py
View File

@@ -116,8 +116,9 @@ cms_obj_li = {
'base_name_alt': Site_Domain_FQDN_ID_Base,
# V3 Search Security:
'searchable_fields': [
'site_domain_id_random', 'account_id_random', 'site_id_random',
'fqdn', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on'
'id', 'account_id', 'site_id',
'id_random', 'account_id_random', 'site_id_random',
'fqdn', 'enable', 'created_on', 'updated_on'
],
},
}

8
app/object_definitions/journals.py
View File

@@ -20,7 +20,9 @@ journal_obj_li = {
],
# V3 Search Security:
'searchable_fields': [
'journal_id_random', 'name', 'short_name', 'summary', 'outline',
'id', 'account_id', 'person_id', 'user_id',
'journal_id_random', 'account_id_random', 'person_id_random', 'user_id_random',
'name', 'short_name', 'summary', 'outline',
'description', 'type_code', 'tags', 'billable', 'enable', 'hide',
'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on'
],
@@ -42,7 +44,9 @@ journal_obj_li = {
],
# V3 Search Security:
'searchable_fields': [
'journal_entry_id_random', 'journal_id_random', 'name', 'short_name',
'id', 'journal_id', 'account_id',
'journal_entry_id_random', 'journal_id_random', 'account_id_random',
'name', 'short_name',
'summary', 'content', 'type_code', 'topic_code', 'category_code',
'tags', 'location', 'billable', 'enable', 'hide', 'priority', 'sort',
'group', 'notes', 'created_on', 'updated_on'