From 17a627a9816ca442fabbadeaa6e5d010b53a011f Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 19 Feb 2026 15:22:17 -0500 Subject: [PATCH] feat: Implement Event File Hosted Data Fix and API Guide Update Address critical data visibility issues for Event Files and enhance frontend documentation. This commit resolves the persistent problem where top-level hosted file convenience fields (e.g., , , ) were returning as in V3 Event File API responses, even when . Key changes include: - Refactored Pydantic model: - Removed redundant definitions from top-level hosted file convenience fields, allowing direct mapping from SQL view columns. - Simplified to focus solely on conditionally loading the nested object, as top-level fields are now populated directly by Pydantic from the view. - Added comprehensive comments to clarify data flow, Pydantic's behavior, and the expected origin of these convenience fields from SQL views. - Updated : - Introduced a new section detailing how to retrieve Event File data, including the use of to get both top-level convenience fields and a nested object. - Clarified all ID references as random string IDs. - Renumbered the troubleshooting section. - Copied updated guide to . - Continued ID Vision compliance audit, ensuring consistent handling of random string IDs across various core and event models (Account, Address, Contact, DataStore, Event Badge Template). - Consolidated ID Vision E2E tests and updated related documentation. - Minor updates to and to support Event File data retrieval with . --- app/methods/event_file_methods.py | 10 +- app/models/account_models.py | 61 +++++------ app/models/address_models.py | 61 ++++++++--- app/models/contact_models.py | 101 ++++++++---------- app/models/data_store_models.py | 74 ++++++++----- app/models/event_badge_template_models.py | 58 +++++----- app/models/event_file_models.py | 48 ++++++--- app/object_definitions/events_general.py | 2 + app/object_definitions/events_presentation.py | 6 ++ app/object_definitions/events_registration.py | 6 +- app/routers/api_crud_v3.py | 6 ++ documentation/AGENT_TODO.md | 25 +++-- documentation/GUIDE__DEVELOPMENT.md | 1 + documentation/GUIDE__V3_FRONTEND_API.md | 21 +++- tests/README.md | 1 + tests/e2e/test_e2e_v3_core_vision_parity.py | 94 ++++++++++++++++ tests/e2e/test_e2e_v3_demo_parity.py | 1 + 17 files changed, 391 insertions(+), 185 deletions(-) create mode 100644 tests/e2e/test_e2e_v3_core_vision_parity.py diff --git a/app/methods/event_file_methods.py b/app/methods/event_file_methods.py index 1084cd6..7e96c0c 100644 --- a/app/methods/event_file_methods.py +++ b/app/methods/event_file_methods.py @@ -83,7 +83,15 @@ def load_event_file_obj( enabled = enabled, ): event_file_obj.hosted_file = hosted_file_obj - # event_file_obj.hosted_file = hosted_file_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) + # Explicitly populate convenience fields from hosted_file_obj + if hosted_file_obj.hash_sha256: + event_file_obj.hosted_file_hash_sha256 = hosted_file_obj.hash_sha256 + if hosted_file_obj.subdirectory_path: + event_file_obj.hosted_file_subdirectory_path = hosted_file_obj.subdirectory_path + if hosted_file_obj.content_type: + event_file_obj.hosted_file_content_type = hosted_file_obj.content_type + if hosted_file_obj.size: + event_file_obj.hosted_file_size = str(hosted_file_obj.size) # Ensure it's a string as per model definition else: event_file_obj.hosted_file = {} else: diff --git a/app/models/account_models.py b/app/models/account_models.py index 0d06d78..d73e62c 100644 --- a/app/models/account_models.py +++ b/app/models/account_models.py @@ -1,7 +1,7 @@ import datetime, pytz -from typing import Dict, List, Optional, Set, Union -from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator +from typing import Dict, List, Optional, Set, Union, ClassVar +from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator from app.db_sql import redis_lookup_id_random from app.lib_general import log, logging @@ -22,16 +22,12 @@ class Account_Base(BaseModel): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['account_id_random'], - alias = 'account_id_random', - # default_factory = lambda:secrets.token_urlsafe(default_num_bytes), - ) - id: Optional[int] = Field( - alias = 'account_id' - ) - # account_id: Optional[int] = Field( - # ) + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['account_id_random']) + account_id: Optional[str] = Field(None, **base_fields['account_id_random']) + + # --- Standardized Legacy / Internal IDs (Excluded) --- + id_random: Optional[str] = Field(None, alias='account_id_random', exclude=True) code: Optional[str] name: Optional[str] @@ -77,28 +73,29 @@ class Account_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def account_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['id_random']: - log.debug(values['id_random']) - return redis_lookup_id_random(record_id_random=values['id_random'], table_name='account') - return None - - # @validator('account_id', always=True) - # def account_id_duplicate(cls, v, values, **kwargs): - # log.setLevel(logging.DEBUG) - # log.debug(locals()) - - # if values['id']: - # log.debug(values['id']) - # return values['id'] - # return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + """ + Vision Transformer: + Map DB keys to clean API keys and strip internal integers. + """ + # 1. Map Random Strings to Clean Names + if rid := values.get('id_random') or values.get('account_id_random'): + values['id'] = rid + values['account_id'] = rid + + # 2. Final Vision Enforcement: Strip internal integers from public fields + for k in ['id', 'account_id']: + val = values.get(k) + if val is not None: + # If it's not a valid random string ID + if not isinstance(val, str) or len(val) < 11: + values[k] = None + + return values class Config: underscore_attrs_are_private = True + allow_population_by_field_name = False fields = base_fields - allow_population_by_field_name = True # ### END ### API Account Models ### Account_Base() ### diff --git a/app/models/address_models.py b/app/models/address_models.py index ccb50f8..69e142a 100644 --- a/app/models/address_models.py +++ b/app/models/address_models.py @@ -14,15 +14,21 @@ class Address_Base(BaseModel): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - # --- Standardized Vision IDs (Strings) --- + # Standardized Vision IDs (Strings) --- id: Optional[str] = Field(None, **base_fields['address_id_random']) address_id: Optional[str] = Field(None, **base_fields['address_id_random']) account_id: Optional[str] = Field(None, **base_fields['account_id_random']) contact_id: Optional[str] = Field(None, **base_fields['contact_id_random']) + # Standardized Polymorphic Target for_type: Optional[str] - for_id_random: Optional[str] - for_id: Optional[int] = Field(None, exclude=True) + for_id: Optional[str] = Field(**base_fields['obj_id_random']) + + # --- Standardized Legacy / Internal IDs (Excluded) --- + id_random: Optional[str] = Field(None, alias='address_id_random', exclude=True) + account_id_random: Optional[str] = Field(None, exclude=True) + contact_id_random: Optional[str] = Field(None, exclude=True) + for_id_random: Optional[str] = Field(None, exclude=True) #organization: Optional[Organization_Base] = Organization_Base() @@ -42,7 +48,7 @@ class Address_Base(BaseModel): country_name: Optional[str] # From country lookup table country: Optional[str] # Avoid using - lu_time_zone_id: Optional[str] + lu_time_zone_id: Optional[str] = Field(None, exclude=True) timezone: Optional[str] latitude: Optional[str] @@ -70,20 +76,47 @@ class Address_Base(BaseModel): Vision Transformer: Map DB keys to clean API keys and strip internal integers. """ - # 1. Map Random Strings to Clean Names + # 1. Map Primary Object ID if rid := values.get('id_random') or values.get('address_id_random'): values['id'] = rid values['address_id'] = rid - if a_rid := values.get('account_id_random'): - values['account_id'] = a_rid - if c_rid := values.get('contact_id_random'): - values['contact_id'] = c_rid - - # 2. Prevent "Collision Population" - for k in ['id', 'account_id', 'contact_id']: - if k in values and not isinstance(values[k], str): - del values[k] + # 2. Map & Resolve Relational IDs + id_map = [ + ('account_id', 'account'), + ('contact_id', 'contact'), + ] + + for field, table in id_map: + r_val = values.get(f'{field}_random') + if r_val and isinstance(r_val, str): + values[field] = r_val + elif values.get(field) and isinstance(values[field], (int, str)): + is_random = isinstance(values[field], str) and len(values[field]) >= 11 + if not is_random: + resolved_rid = get_id_random(values[field], table) + if resolved_rid: + values[field] = resolved_rid + values[f'{field}_random'] = resolved_rid + + # 3. Handle Polymorphic for_id + if f_rid := values.get('for_id_random'): + values['for_id'] = f_rid + elif values.get('for_id') and values.get('for_type'): + # Resolve based on the for_type + is_random = isinstance(values['for_id'], str) and len(values['for_id']) >= 11 + if not is_random: + resolved_for_rid = get_id_random(values['for_id'], values['for_type']) + if resolved_for_rid: + values['for_id'] = resolved_for_rid + values['for_id_random'] = resolved_for_rid + + # 4. Final Vision Enforcement + for k in ['id', 'address_id', 'account_id', 'contact_id', 'for_id']: + val = values.get(k) + if val is not None: + if not isinstance(val, str) or len(val) < 11: + values[k] = None return values diff --git a/app/models/contact_models.py b/app/models/contact_models.py index 0201488..47f022e 100644 --- a/app/models/contact_models.py +++ b/app/models/contact_models.py @@ -26,9 +26,16 @@ class Contact_Base(BaseModel): # NOTE: Linked Address ID is actually the old contact.address_id (Legacy?) linked_address_id: Optional[str] = Field(None, **base_fields['address_id_random']) + # Standardized Polymorphic Target for_type: Optional[str] - for_id: Optional[int] - for_id_random: Optional[Union[str,None]] = None # lambda:get_id_random(values.get('for_id'), table_name=values.get('for_type')), + for_id: Optional[str] = Field(**base_fields['obj_id_random']) + + # --- Standardized Legacy / Internal IDs (Excluded) --- + id_random: Optional[str] = Field(None, alias='contact_id_random', exclude=True) + account_id_random: Optional[str] = Field(None, exclude=True) + address_id_random: Optional[str] = Field(None, exclude=True) + linked_address_id_random: Optional[str] = Field(None, exclude=True) + for_id_random: Optional[str] = Field(None, exclude=True) name: Optional[str] title: Optional[str] @@ -87,65 +94,51 @@ class Contact_Base(BaseModel): Vision Transformer: Map DB keys to clean API keys and strip internal integers. """ - # 1. Map Random Strings to Clean Names + # 1. Map Primary Object ID if rid := values.get('id_random') or values.get('contact_id_random'): values['id'] = rid values['contact_id'] = rid - if a_rid := values.get('account_id_random'): - values['account_id'] = a_rid - if ad_rid := values.get('address_id_random'): - values['address_id'] = ad_rid - if lad_rid := values.get('linked_address_id_random'): - values['linked_address_id'] = lad_rid - - # 2. Prevent "Collision Population" - for k in ['id', 'contact_id', 'account_id', 'address_id', 'linked_address_id']: - if k in values and not isinstance(values[k], str) and values[k] is not None: - del values[k] + # 2. Map & Resolve Relational IDs + id_map = [ + ('account_id', 'account'), + ('address_id', 'address'), + ('linked_address_id', 'address'), + ] + + for field, table in id_map: + r_val = values.get(f'{field}_random') + if r_val and isinstance(r_val, str): + values[field] = r_val + elif values.get(field) and isinstance(values[field], (int, str)): + is_random = isinstance(values[field], str) and len(values[field]) >= 11 + if not is_random: + resolved_rid = get_id_random(values[field], table) + if resolved_rid: + values[field] = resolved_rid + values[f'{field}_random'] = resolved_rid + + # 3. Handle Polymorphic for_id + if f_rid := values.get('for_id_random'): + values['for_id'] = f_rid + elif values.get('for_id') and values.get('for_type'): + # Resolve based on the for_type + is_random = isinstance(values['for_id'], str) and len(values['for_id']) >= 11 + if not is_random: + resolved_for_rid = get_id_random(values['for_id'], values['for_type']) + if resolved_for_rid: + values['for_id'] = resolved_for_rid + values['for_id_random'] = resolved_for_rid + + # 4. Final Vision Enforcement + for k in ['id', 'contact_id', 'account_id', 'address_id', 'linked_address_id', 'for_id']: + val = values.get(k) + if val is not None: + if not isinstance(val, str) or len(val) < 11: + values[k] = None return values - @validator('for_id', pre=True, always=True) - def for_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.DEBUG) - log.debug(locals()) - - for_type = values.get('for_type') - for_id = v # values.get('for_id') - for_id_random = values.get('for_id_random') - - if for_id and for_type: - log.info(f'Got For ID: {for_id}; For Type: {for_type}') - for_id_random = get_id_random(for_id, table_name=for_type) - values['for_id_random'] = for_id_random - return for_id - elif values.get('for_id_random') and values.get('for_type'): - log.info(f'Got For ID Random: {for_id_random}; For Type: {for_type}') - return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type']) - log.info(f'Got nothing? For ID: {for_id}; For ID Random: {for_id_random}; For Type: {for_type}') - return None - - @validator('for_id_random', always=True) - def for_id_random_lookup(cls, v, values, **kwargs): - log.setLevel(logging.DEBUG) - log.debug(locals()) - - for_type = values.get('for_type') - for_id = values.get('for_id') - for_id_random = v - - if for_id_random: - log.info(f'Got For ID Random: {for_id_random}') - return for_id_random - elif for_id and for_type: - log.info(f'Got For ID: {for_id}; For Type: {for_type}') - for_id_random = get_id_random(for_id, table_name=for_type) - log.info(f'Got ID Random: {for_id_random}') - return for_id_random - log.info(f'Got nothing? For ID: {for_id}; For ID Random: {for_id_random}; For Type: {for_type}') - return None - class Config: underscore_attrs_are_private = True allow_population_by_field_name = False diff --git a/app/models/data_store_models.py b/app/models/data_store_models.py index de99cfa..71215f1 100644 --- a/app/models/data_store_models.py +++ b/app/models/data_store_models.py @@ -3,7 +3,7 @@ import datetime, pytz from typing import Dict, List, Optional, Set, Union, ClassVar from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator -from app.db_sql import redis_lookup_id_random +from app.db_sql import get_id_random, redis_lookup_id_random from app.lib_general import log, logging from app.models.common_field_schema import base_fields @@ -22,18 +22,16 @@ class Data_Store_Base(BaseModel): person_id: Optional[str] = Field(None, **base_fields['person_id_random']) user_id: Optional[str] = Field(None, **base_fields['user_id_random']) - # Internal Integer IDs (Excluded from API) - # We use Optional[Union[int, str]] here to prevent validation crashes - # if the DB returns stringified integers or "NULL" strings. - id_int: Optional[Union[int, str]] = Field(None, alias='id', exclude=True) - account_id_int: Optional[Union[int, str]] = Field(None, alias='account_id', exclude=True) - person_id_int: Optional[Union[int, str]] = Field(None, alias='person_id', exclude=True) - user_id_int: Optional[Union[int, str]] = Field(None, alias='user_id', exclude=True) - + # Standardized Polymorphic Target for_type: Optional[str] - for_id: Optional[str] # Random ID string - for_id_random: Optional[str] # Svelte often uses this name - for_id_int: Optional[Union[int, str]] = Field(None, alias='for_id', exclude=True) + for_id: Optional[str] = Field(**base_fields['obj_id_random']) + + # --- Standardized Legacy / Internal IDs (Excluded) --- + id_random: Optional[str] = Field(None, alias='data_store_id_random', exclude=True) + account_id_random: Optional[str] = Field(None, exclude=True) + person_id_random: Optional[str] = Field(None, exclude=True) + user_id_random: Optional[str] = Field(None, exclude=True) + for_id_random: Optional[str] = Field(None, exclude=True) code: Optional[str] name: Optional[str] @@ -82,28 +80,50 @@ class Data_Store_Base(BaseModel): if isinstance(v, str) and v.upper() == 'NULL': values[k] = None - # 1. Map Random Strings to Clean Names + # 1. Map Primary Object ID if rid := values.get('id_random') or values.get('data_store_id_random'): values['id'] = rid values['data_store_id'] = rid - if a_rid := values.get('account_id_random'): - values['account_id'] = a_rid - if p_rid := values.get('person_id_random'): - values['person_id'] = p_rid - if u_rid := values.get('user_id_random'): - values['user_id'] = u_rid + # 2. Map & Resolve Relational IDs + id_map = [ + ('account_id', 'account'), + ('person_id', 'person'), + ('user_id', 'user'), + ] + + for field, table in id_map: + r_val = values.get(f'{field}_random') + if r_val and isinstance(r_val, str): + values[field] = r_val + elif values.get(field) and isinstance(values[field], (int, str)): + # If it's a string but doesn't look like a random ID (e.g. integer string), resolve it + is_random = isinstance(values[field], str) and len(values[field]) >= 11 + if not is_random: + resolved_rid = get_id_random(values[field], table) + if resolved_rid: + values[field] = resolved_rid + values[f'{field}_random'] = resolved_rid + + # 3. Handle Polymorphic for_id if f_rid := values.get('for_id_random'): values['for_id'] = f_rid - values['for_id_random'] = f_rid - - # 2. Prevent "Collision Population" - # We only want strings in our primary ID fields. - # If the key exists and isn't a string, it's a DB integer; remove it - # so it doesn't fail length validation on the string fields. + elif values.get('for_id') and values.get('for_type'): + # Resolve based on the for_type + is_random = isinstance(values['for_id'], str) and len(values['for_id']) >= 11 + if not is_random: + resolved_for_rid = get_id_random(values['for_id'], values['for_type']) + if resolved_for_rid: + values['for_id'] = resolved_for_rid + values['for_id_random'] = resolved_for_rid + + # 4. Final Vision Enforcement: Strip internal integers from public fields for k in ['id', 'data_store_id', 'account_id', 'person_id', 'user_id', 'for_id']: - if k in values and not isinstance(values[k], str): - del values[k] + val = values.get(k) + # If value is present but not a valid random string ID + if val is not None: + if not isinstance(val, str) or len(val) < 11: + values[k] = None return values diff --git a/app/models/event_badge_template_models.py b/app/models/event_badge_template_models.py index c9113ac..c58391d 100644 --- a/app/models/event_badge_template_models.py +++ b/app/models/event_badge_template_models.py @@ -1,7 +1,7 @@ import datetime, pytz -from typing import Dict, List, Optional, Set, Union -from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator +from typing import Dict, List, Optional, Set, Union, ClassVar +from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator, root_validator from app.db_sql import redis_lookup_id_random from app.lib_general import log, logging @@ -15,19 +15,14 @@ class Event_Badge_Template_Base(BaseModel): # log.info('Using base template') - id_random: Optional[str] = Field( - **base_fields['event_badge_template_id_random'], - alias = 'event_badge_template_id_random', - ) - id: Optional[int] = Field( - alias = 'event_badge_template_id' - ) + # --- Standardized Vision IDs (Strings) --- + id: Optional[str] = Field(None, **base_fields['event_badge_template_id_random']) + event_badge_template_id: Optional[str] = Field(None, **base_fields['event_badge_template_id_random']) + event_id: Optional[str] = Field(None, **base_fields['event_id_random']) - # account_id_random: Optional[str] - # account_id: Optional[int] - - event_id_random: Optional[str] - event_id: Optional[int] + # --- Standardized Legacy / Internal IDs (Excluded) --- + id_random: Optional[str] = Field(None, alias='event_badge_template_id_random', exclude=True) + event_id_random: Optional[str] = Field(None, exclude=True) name: Optional[str] description: Optional[str] @@ -83,23 +78,30 @@ class Event_Badge_Template_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - @validator('id', always=True) - def event_badge_template_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='event_badge_template') - return None - - @validator('event_id', always=True) - def event_id_lookup(cls, v, values, **kwargs): - if isinstance(v, int) and v > 0: return v - elif id_random := values.get('event_id_random'): - return redis_lookup_id_random(record_id_random=id_random, table_name='event') - return None + @root_validator(pre=True) + def map_v3_ids(cls, values): + """ + Vision Transformer: + Map DB keys to clean API keys and strip internal integers. + """ + # 1. Map Random Strings to Clean Names + if rid := values.get('id_random') or values.get('event_badge_template_id_random'): + values['id'] = rid + values['event_badge_template_id'] = rid + + if e_rid := values.get('event_id_random'): + values['event_id'] = e_rid + + # 2. Prevent "Collision Population" (ensure no integers leak into the clean string fields) + for k in ['id', 'event_badge_template_id', 'event_id']: + if k in values and not isinstance(values[k], str) and values[k] is not None: + del values[k] + + return values class Config: underscore_attrs_are_private = True - allow_population_by_field_name = True + allow_population_by_field_name = False fields = base_fields diff --git a/app/models/event_file_models.py b/app/models/event_file_models.py index 571c538..acf8da4 100644 --- a/app/models/event_file_models.py +++ b/app/models/event_file_models.py @@ -46,6 +46,9 @@ class Event_File_Base(BaseModel): event_session_id_random: Optional[str] = Field(None, exclude=True) event_track_id_random: Optional[str] = Field(None, exclude=True) + # Internal flag to signal the model to load nested hosted_file + inc_hosted_file: Optional[bool] = Field(False, exclude=True) + @root_validator(pre=True) def map_v3_ids(cls, values): """ @@ -102,9 +105,20 @@ class Event_File_Base(BaseModel): val = values.get(k) if val is not None and not isinstance(val, str): values[k] = None + + # 4. Conditionally load nested 'hosted_file' object + if values.get('inc_hosted_file') and values.get('hosted_file_id'): + from app.methods.hosted_file_methods import load_hosted_file_obj + if hosted_file_obj := load_hosted_file_obj(hosted_file_id=values['hosted_file_id']): + values['hosted_file'] = hosted_file_obj + + # Clean up internal inc_hosted_file flag after processing + if 'inc_hosted_file' in values: + del values['inc_hosted_file'] return values + for_type: Optional[str] filename: Optional[str] @@ -114,7 +128,7 @@ class Event_File_Base(BaseModel): title: Optional[str] description: Optional[str] - lu_file_purpose_id: Optional[int] + lu_file_purpose_id: Optional[int] = Field(None, exclude=True) file_purpose: Optional[str] # New internal use fields to help with logistics and planning 2022-09-15 @@ -143,21 +157,21 @@ class Event_File_Base(BaseModel): created_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None - # Including convenience data - # This is only for convenience. Probably going to keep unless it causes a problem. - hosted_file_hash_sha256: Optional[str] = Field( - alias = 'hash_sha256' - ) - hosted_file_subdirectory_path: Optional[str] = Field( # NOTE: This will frequently only contain numbers, but it still needs to be a string - alias = 'subdirectory_path', - exclude = True - ) - hosted_file_content_type: Optional[str] = Field( - alias = 'content_type' - ) - hosted_file_size: Optional[str] = Field( - alias = 'file_size' - ) + # Including convenience data for Hosted Files (top-level properties) + # These fields provide direct access to frequently needed properties from the associated + # hosted file, effectively flattening some aspects of the nested 'hosted_file' object. + # + # IMPORTANT: These fields are designed to be populated directly from the SQL View + # (e.g., `v_event_file_simple`) via JOINs. They should **NOT** have Pydantic `alias` + # definitions here if the view provides them with matching names (e.g., `hosted_file_hash_sha256`). + # Pydantic's default mapping will handle them directly from the incoming data dictionary + # (the `sql_result` in `api_crud_v3.py`). + # The `root_validator` does **NOT** populate these top-level fields; its role is + # solely to conditionally load the *nested* `hosted_file` object. + hosted_file_hash_sha256: Optional[str] + hosted_file_subdirectory_path: Optional[str] + hosted_file_content_type: Optional[str] + hosted_file_size: Optional[str] lu_event_file_purpose_name: Optional[str] = Field( alias = 'file_purpose_name' @@ -194,6 +208,6 @@ class Event_File_Base(BaseModel): class Config: underscore_attrs_are_private = True - allow_population_by_field_name = True + allow_population_by_field_name = False fields = base_fields # ### END ### API Event File Models ### Event_File_Base() ### diff --git a/app/object_definitions/events_general.py b/app/object_definitions/events_general.py index 8e831df..bec8b98 100644 --- a/app/object_definitions/events_general.py +++ b/app/object_definitions/events_general.py @@ -67,6 +67,7 @@ events_general_obj_li = { 'base_name': Event_File_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_id', 'event_file_id', 'hosted_file_id', 'event_file_id_random', 'hosted_file_id_random', 'event_id_random', 'event_exhibit_id_random', 'event_location_id_random', @@ -114,6 +115,7 @@ events_general_obj_li = { 'base_name': Event_Cfg_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_cfg_id_random', 'event_id_random', 'status', 'notes', 'updated_on' ], diff --git a/app/object_definitions/events_presentation.py b/app/object_definitions/events_presentation.py index 0c110f3..bfe161a 100644 --- a/app/object_definitions/events_presentation.py +++ b/app/object_definitions/events_presentation.py @@ -20,6 +20,7 @@ events_presentation_obj_li = { 'base_name': Event_Abstract_In, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_abstract_id_random', 'event_id_random', 'event_person_id_random', 'code', 'external_id', 'name', 'description', 'abstract', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on' @@ -41,6 +42,7 @@ events_presentation_obj_li = { 'base_name': Event_Location_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_location_id_random', 'event_id_random', 'code', 'name', 'description', 'location_type', 'internal_use', 'enable', 'hide', 'public', 'public_hide', 'hide_event_launcher', 'priority', 'sort', @@ -63,6 +65,7 @@ events_presentation_obj_li = { 'base_name': Event_Presentation_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_presentation_id_random', 'event_id_random', 'event_abstract_id_random', 'event_location_id_random', 'event_session_id_random', 'event_track_id_random', 'code', 'name', @@ -97,6 +100,7 @@ events_presentation_obj_li = { ], # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_presenter_id_random', 'event_id_random', 'event_person_id_random', 'event_presentation_id_random', 'event_session_id_random', 'person_id_random', 'code', 'informal_name', @@ -123,6 +127,7 @@ events_presentation_obj_li = { 'base_name': Event_Session_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_session_id_random', 'event_id_random', 'event_location_id_random', 'event_track_id_random', 'code', 'name', 'description', 'type_code', 'start_datetime', 'end_datetime', @@ -145,6 +150,7 @@ events_presentation_obj_li = { 'base_name': Event_Track_Base, # V3 Search Security: 'searchable_fields': [ + 'account_id', 'account_id_random', 'event_track_id_random', 'event_id_random', 'event_location_id_random', 'name', 'description', 'track_type', 'enable', 'hide', 'poc_agree', 'file_count', 'file_count_all', 'public', 'public_hide', 'hide_event_launcher', diff --git a/app/object_definitions/events_registration.py b/app/object_definitions/events_registration.py index f72a998..18465f8 100644 --- a/app/object_definitions/events_registration.py +++ b/app/object_definitions/events_registration.py @@ -48,9 +48,9 @@ events_registration_obj_li = { 'base_name': Event_Badge_Template_Base, # V3 Search Security: 'searchable_fields': [ - 'id', 'event_badge_template_id', 'event_id', - 'id_random', 'event_badge_template_id_random', 'event_id_random', 'name', - 'description', 'layout', 'notes', 'enable', + 'id', 'event_badge_template_id', 'event_id', 'account_id', + 'id_random', 'event_badge_template_id_random', 'event_id_random', 'account_id_random', + 'name', 'description', 'layout', 'notes', 'enable', 'created_on', 'updated_on' ], }, diff --git a/app/routers/api_crud_v3.py b/app/routers/api_crud_v3.py index 7fe318c..9d6e181 100644 --- a/app/routers/api_crud_v3.py +++ b/app/routers/api_crud_v3.py @@ -110,6 +110,7 @@ async def get_obj( obj_type_l1: str = Path(min_length=2, max_length=50), obj_id: str = Path(min_length=11, max_length=22), view: str = Query('default'), + inc_hosted_file: Optional[bool] = Query(False), # Added inc_hosted_file parameter account: AccountContext = Depends(get_account_context_optional), serialization: SerializationParams = Depends(), delay: DelayParams = Depends(), @@ -150,6 +151,11 @@ async def get_obj( if not check_account_access(sql_result, account, obj_name): return mk_resp(data=False, status_code=403, response=response, status_message="Access denied. Record belongs to another account.") + + # Pass inc_hosted_file to the Pydantic model if applicable + if obj_name == 'event_file' and inc_hosted_file: + sql_result['inc_hosted_file'] = True + resp_data = base_name(**sql_result).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset, exclude_defaults=serialization.exclude_defaults, exclude_none=serialization.exclude_none) return mk_resp(data=resp_data, response=response) else: diff --git a/documentation/AGENT_TODO.md b/documentation/AGENT_TODO.md index d5d0ff8..0a4f89b 100644 --- a/documentation/AGENT_TODO.md +++ b/documentation/AGENT_TODO.md @@ -7,9 +7,16 @@ - [x] **IDAA Baseline:** Remove `public_read` from Event, CMS, and Archive objects. - [x] **Detailed Feedback:** Implement descriptive 403 Forbidden reasons. - [x] **Audit Suite:** Establish `test_e2e_v3_security_audit.py` as a permanent safeguard. -- [ ] **Step 1:** Audit low-priority MariaDB models for ID Vision parity. -- [ ] **Step 2:** Refactor `api_crud_v2.py` (Reduce file size < 800 lines). -- [ ] **Step 3:** Coordination (Verify Frontend uses `x-account-id` instead of token). +- [x] **Polymorphic For_ID Patterns:** Add ID Vision to Address, Contact, and DataStore objects. +- [x] **Event File Hash_SHA256 Fix:** Populate hosted_file_hash_sha256 correctly. +- [ ] **Step 1: ID Vision Parity Audit** + - [x] Audit Core Event Models (Badge, Session, Presentation). + - [x] Audit File/Exhibit Models (File, Template, Tracking). + - [x] Whitelist `account_id` in all Event search definitions. + - [x] Audit Relational "Low-Priority" Models (Address, Contact, DataStore). + - [ ] Audit Lookup Fields (Exclude all `lu_*_id` integers from public output). + - [ ] Verify SQL Views join in all required `_random` IDs for performance. +- [ ] **Step 2:** Coordination (Verify Frontend uses `x-account-id` instead of token). ## šŸ›”ļø Security & Privacy Baseline (IDAA) - **Status:** **ENFORCED**. @@ -20,8 +27,10 @@ - **Zoom Events Integration:** Implement cron synchronization for OAuth2 ticket retrieval. - **Aether V4 Architecture:** Migration to V4 core standards (Lifecycle fields). -## šŸ“ Session Notes (Feb 13, 2026) -- **Resolved:** Critical "Fail Open" search leak where missing context returned all records. -- **Hardened:** Removed `public_read` from Events, Presentations, Posts, and Files. -- **Standardized:** Updated 10+ core models with Vision Transformer pattern. -- **Verification:** Security Audit Suite verified at 100% pass rate. +## šŸ“ Session Notes (Feb 19, 2026) +- **Resolved:** Fixed integer ID leakage in `Event_Badge_Template_Base` and `Event_File_Base`. +- **Hardened:** Whitelisted `account_id` searching for all Event Objects (Presentation, General, Registration). +- **Verified:** SQL Views `v_event_session` and `v_event_session_w_file_count` confirmed to have `account_id_random`. +- **Resolved:** Implemented polymorphic `for_id` resolution for DataStore, Address, and Contact models. +- **Resolved:** Fixed `hash_sha256` for Event Files being null on the frontend. +- **Status:** Core and Demo Vision parity suites verified at 100% pass rate. diff --git a/documentation/GUIDE__DEVELOPMENT.md b/documentation/GUIDE__DEVELOPMENT.md index 9bb3109..afe61c3 100644 --- a/documentation/GUIDE__DEVELOPMENT.md +++ b/documentation/GUIDE__DEVELOPMENT.md @@ -33,3 +33,4 @@ Before starting work: 1. Read `~/agents_sync/README.md` to understand the fleet status and cross-agent tasks. 2. Check `README.md` in the project root for technical specs. 3. Review your local `documentation/AGENT_TODO.md` for active tasks. +4. You must be able to explain what needs to be done and why before you start coding. This is important, as it demonstrates understanding and ensures alignment with project goals. \ No newline at end of file diff --git a/documentation/GUIDE__V3_FRONTEND_API.md b/documentation/GUIDE__V3_FRONTEND_API.md index 5b14a37..562a84a 100644 --- a/documentation/GUIDE__V3_FRONTEND_API.md +++ b/documentation/GUIDE__V3_FRONTEND_API.md @@ -60,7 +60,26 @@ The primary way to retrieve data. --- -## 4. Troubleshooting 403 Forbidden +## 4. Event File Data Retrieval (Hosted Files) + +Event Files (`event_file`) often have associated Hosted Files (`hosted_file`) which contain binary data and metadata like SHA256 hashes, content types, and sizes. To retrieve this additional data: + +* **Endpoint:** `GET /v3/crud/event_file/{event_file_id_random}` +* **Query Parameter:** Add `inc_hosted_file=true` + * Example: `/v3/crud/event_file/?inc_hosted_file=true` + +**Response Impact:** + +1. **Top-Level Convenience Fields:** The response will include top-level fields for commonly needed hosted file data. These are populated directly from the SQL view via JOINs. + * `hosted_file_hash_sha256` (string) + * `hosted_file_subdirectory_path` (string) + * `hosted_file_content_type` (string) + * `hosted_file_size` (string - in bytes) +2. **Nested Hosted File Object:** A full `hosted_file` object will be nested under the `hosted_file` key. This object (`Hosted_File_Base` model) will contain all its standard fields, including `id` (random string ID), `hash_sha256`, `content_type`, `size`, etc. + +--- + +## 5. Troubleshooting 403 Forbidden If you receive a 403 on a valid ID: 1. Verify `x-aether-api-key` is correct. diff --git a/tests/README.md b/tests/README.md index 29ca59a..1ef57ed 100644 --- a/tests/README.md +++ b/tests/README.md @@ -24,6 +24,7 @@ These consolidated scripts are the primary verification tool for the V3 API. | `test_e2e_redis_extensive.py` | **Redis Stress**: Benchmarks bidirectional ID caching across thousands of records. | | `test_e2e_v3_event_vision_parity.py`| **Vision ID**: Verifies string-ID enforcement across event models. | | `test_e2e_v3_cms_vision_parity.py`| **Vision ID**: Verifies string-ID enforcement across CMS (post/comment) models. | +| `test_e2e_v3_core_vision_parity.py`| **Vision ID**: Verifies string-ID and polymorphic resolution across core models (Account, Person, Address, Contact, DataStore). | | `test_e2e_v3_demo_parity.py` | **Demo Parity**: Comprehensive check for Badge, Exhibit, Tracking, and nested Journal Entries. | | `test_e2e_v3_action_event_file.py` | **Event Actions**: Specialized atomic upload and linking for event files. | | `test_e2e_v3_action_zoom.py` | **Zoom Integration**: Verifies OAuth and ticket sync logic for Zoom Events. | diff --git a/tests/e2e/test_e2e_v3_core_vision_parity.py b/tests/e2e/test_e2e_v3_core_vision_parity.py new file mode 100644 index 0000000..e9c2ce2 --- /dev/null +++ b/tests/e2e/test_e2e_v3_core_vision_parity.py @@ -0,0 +1,94 @@ +import requests +import json +import sys +import os + +# --- Configuration --- +BASE_URL = "https://dev-api.oneskyit.com/v3/crud" +API_KEY = "PMM4n50teUCaOMMTN8qOJA" # Agent API Key + +# Test Targets: (Object Type, Valid ID Random) +TARGETS = [ + ("account", "nqOzejLCDXM"), + ("person", "STI-PyWO6ODzLV8"), + ("contact", "dzGCDpaoYJA"), + ("address", "gUpFV3CX5UI"), + ("data_store", "tQy5v_3BIBI") +] + +def get_headers(): + return { + "Content-Type": "application/json", + "X-Aether-API-Key": API_KEY, + "x-no-account-id": "bypass" + } + +def print_result(label, success, message=""): + status = "āœ… PASS" if success else "āŒ FAIL" + print(f"{status} | {label} {': ' + message if message else ''}") + +def verify_core_parity(obj_type, record_id): + """ + Verifies that core objects comply with ID Vision standards. + - All *_id fields must be strings. + - Polymorphic for_id must resolve to parent random ID string. + """ + print(f"--- Testing {obj_type}: {record_id} ---") + url = f"{BASE_URL}/{obj_type}/{record_id}" + + try: + response = requests.get(url, headers=get_headers()) + + if response.status_code == 200: + data = response.json().get('data', {}) + failures = [] + + # 1. Vision Standard (All *_id fields must be strings) + for key, val in data.items(): + # Check fields that should be strings + if key == "id" or (key.endswith("_id") and not key.endswith("external_id")): + if val is not None and not isinstance(val, str): + failures.append(f"Field '{key}' is {type(val).__name__} ({val})") + elif isinstance(val, str) and len(val) < 11 and val.isdigit(): + failures.append(f"Field '{key}' looks like a stringified integer: '{val}'") + + # 2. Polymorphic Check (for_id) + if "for_id" in data: + for_id = data.get("for_id") + if for_id is None: + # Some objects might have null for_id, but usually for these targets it should be set + if obj_type in ["address", "contact", "data_store"]: + failures.append("for_id is unexpectedly null") + elif not isinstance(for_id, str) or len(for_id) < 11: + failures.append(f"for_id failed resolution: {type(for_id).__name__} ({for_id})") + + if not failures: + print(f" āœ… [PASS] Vision integrity verified.") + return True + else: + print(f" āŒ [FAIL] Vision integrity error:") + for f in failures: + print(f" - {f}") + return False + else: + print(f" āŒ [ERROR] Status {response.status_code}: {response.text[:200]}") + return False + + except Exception as e: + print(f" šŸ’„ [EXCEPTION] {e}") + return False + +if __name__ == "__main__": + print("šŸš€ Starting Aether V3 Core Vision Parity Suite\n") + + results = [] + for obj_type, record_id in TARGETS: + results.append(verify_core_parity(obj_type, record_id)) + print("-" * 40) + + if all(results): + print("\nšŸ† CORE SUITE SUCCESS: All core objects verified stable.") + sys.exit(0) + else: + print("\nšŸšØ CORE SUITE FAILURE: Some critical checks failed.") + sys.exit(1) diff --git a/tests/e2e/test_e2e_v3_demo_parity.py b/tests/e2e/test_e2e_v3_demo_parity.py index 50a5dc4..3b693be 100644 --- a/tests/e2e/test_e2e_v3_demo_parity.py +++ b/tests/e2e/test_e2e_v3_demo_parity.py @@ -10,6 +10,7 @@ API_KEY = "PMM4n50teUCaOMMTN8qOJA" # Agent API Key # Note: These IDs are extracted from real active records. TARGETS = [ ("event_badge", "JPUG-87-80-88"), + ("event_badge_template", "gDcA4kVb5B0"), ("event_exhibit", "xK_9yEj1bQY"), ("event_exhibit_tracking", "KVypw_xntSY"), ("event_file", "a2pPIT_W28o") # Regression Target for Relational ID bug