From 7b9ec69e7bea2875be0df371e0f412d6b97cb8b5 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Fri, 2 Jan 2026 16:14:41 -0500 Subject: [PATCH] Refactor: Add legacy V2 support to modern object definitions and document V3 architecture. --- app/ae_obj_types_def.py | 176 +++++- app/routers/api_crud_v3.py | 565 +++++++++++++++++- .../V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md | 53 ++ 3 files changed, 760 insertions(+), 34 deletions(-) create mode 100644 documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md diff --git a/app/ae_obj_types_def.py b/app/ae_obj_types_def.py index 6c1b81d..68b37fc 100644 --- a/app/ae_obj_types_def.py +++ b/app/ae_obj_types_def.py @@ -67,6 +67,10 @@ obj_type_kv_li = { 'mdl_default': Activity_Log_Base, 'mdl_in': Activity_Log_Base, 'mdl_out': Activity_Log_Base, + # Legacy V2 keys: + 'table_name': 'v_activity_log', + 'tbl_name_update': 'activity_log', + 'base_name': Activity_Log_Base, }, 'sponsorship': { 'tbl': 'sponsorship', @@ -76,6 +80,10 @@ obj_type_kv_li = { 'mdl_default': Sponsorship_Base, 'mdl_in': Sponsorship_Base, 'mdl_out': Sponsorship_Base, + # Legacy V2 keys: + 'table_name': 'v_sponsorship', + 'tbl_name_update': 'sponsorship', + 'base_name': Sponsorship_Base, 'exp_default': [ 'sponsorship_id_random', # 'account_id_random', 'sponsorship_cfg_id_random', @@ -118,6 +126,10 @@ obj_type_kv_li = { 'mdl_default': Sponsorship_Cfg_Base, # 'mdl_in': None, # 'mdl_out': None + # Legacy V2 keys: + 'table_name': 'v_sponsorship_cfg', + 'tbl_name_update': 'sponsorship_cfg', + 'base_name': Sponsorship_Cfg_Base, }, # Updated 2024-08-14 'event': { @@ -127,11 +139,12 @@ obj_type_kv_li = { 'mdl': Event_Base, 'mdl_default': Event_Base, 'mdl_alt': Event_Meeting_Flat_Base, - # 'table_name': 'v_event', - # 'table_name_alt': 'v_event_w_file_count', - # 'tbl_name_update': 'event', - # 'base_name': Event_Base, - # 'base_name_alt': Event_Meeting_Flat_Base + # Legacy V2 keys: + 'table_name': 'v_event', + 'table_name_alt': 'v_event_w_file_count', + 'tbl_name_update': 'event', + 'base_name': Event_Base, + 'base_name_alt': Event_Meeting_Flat_Base, 'exp_default': [ 'event_id_random', # 'external_person_id', @@ -173,6 +186,11 @@ obj_type_kv_li = { 'mdl': Event_Badge_Base, 'mdl_default': Event_Badge_Basic_Base, # 'mdl_alt': Event_Badge_Basic_Base + # Legacy V2 keys: + 'table_name': 'v_event_badge', + 'table_name_alt': 'v_event_badge_only', + 'tbl_name_update': 'event_badge', + 'base_name': Event_Badge_Basic_Base, }, 'event_badge_template': { 'tbl': 'event_badge_template', @@ -182,6 +200,10 @@ obj_type_kv_li = { 'mdl_default': Event_Badge_Template_Base, # 'mdl_in': Event_Badge_Template_In_Base, # 'mdl_out': Event_Badge_Template_Out_Base + # Legacy V2 keys: + 'table_name': 'v_event_badge_template', + 'tbl_name_update': 'event_badge_template', + 'base_name': Event_Badge_Template_Base, }, # Updated 2024-08-14 'event_device': { @@ -190,6 +212,10 @@ obj_type_kv_li = { 'tbl_alt': 'v_event_device', 'mdl': Event_Device_Base, 'mdl_default': Event_Device_Base, + # Legacy V2 keys: + 'table_name': 'v_event_device', + 'tbl_name_update': 'event_device', + 'base_name': Event_Device_Base, }, # Updated 2024-08-14 'event_file': { @@ -198,10 +224,11 @@ obj_type_kv_li = { 'tbl_alt': 'v_event_file', # or 'v_event_file_detail' 'mdl': Event_File_Base, 'mdl_default': Event_File_Base, - # 'table_name': 'v_event_file_simple', - # 'table_name_alt': 'v_event_file', - # 'tbl_name_update': 'event_file_simple', - # 'base_name': Event_File_Base + # Legacy V2 keys: + 'table_name': 'v_event_file_simple', + 'table_name_alt': 'v_event_file', + 'tbl_name_update': 'event_file', + 'base_name': Event_File_Base, }, # Should this eventually be changed to event_hosted_file # Updated 2024-08-14 'event_location': { @@ -210,10 +237,11 @@ obj_type_kv_li = { 'tbl_alt': 'v_event_location_w_file_count', 'mdl': Event_Location_Base, 'mdl_default': Event_Location_Base, - # 'table_name': 'v_event_location', - # 'table_name_alt': 'v_event_location_w_file_count', - # 'tbl_name_update': 'event_location', - # 'base_name': Event_Location_Base + # Legacy V2 keys: + 'table_name': 'v_event_location', + 'table_name_alt': 'v_event_location_w_file_count', + 'tbl_name_update': 'event_location', + 'base_name': Event_Location_Base, }, # Updated 2024-08-14 'event_presentation': { @@ -222,10 +250,11 @@ obj_type_kv_li = { 'tbl_alt': 'v_event_presentation_w_file_count', 'mdl': Event_Presentation_Base, 'mdl_default': Event_Presentation_Base, - # 'table_name': 'v_event_presentation', - # 'table_name_alt': 'v_event_presentation_w_file_count', - # 'tbl_name_update': 'event_presentation', - # 'base_name': Event_Presentation_Base + # Legacy V2 keys: + 'table_name': 'v_event_presentation', + 'table_name_alt': 'v_event_presentation_w_file_count', + 'tbl_name_update': 'event_presentation', + 'base_name': Event_Presentation_Base, }, # Updated 2024-08-14 'event_presenter': { @@ -237,6 +266,11 @@ obj_type_kv_li = { 'mdl_default': Event_Presenter_Base, 'mdl_in': Event_Presenter_Base, 'mdl_out': Event_Presenter_Out_Base, + # Legacy V2 keys: + 'table_name': 'v_event_presenter', + 'table_name_alt': 'v_event_presenter_w_file_count', + 'tbl_name_update': 'event_presenter', + 'base_name': Event_Presenter_Base, 'exp_default': [ 'event_presenter_id_random', # 'account_id_random', @@ -249,12 +283,6 @@ obj_type_kv_li = { 'comments', 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on', ], - # WARNING: These must be updated soon! - # 'table_name': 'v_event_presenter', - # 'table_name_alt': 'v_event_presenter_w_file_count', - # 'tbl_name_update': 'event_presenter', - # 'base_name': Event_Presenter_Base - # WARNING: These must be updated soon! }, # Updated 2024-08-14 'event_session': { @@ -265,8 +293,10 @@ obj_type_kv_li = { 'mdl': Event_Session_Base, 'mdl_default': Event_Session_Base, 'exclude_for_db': {'poc_person_id', 'file_count', 'internal_use_count', 'enable_from', 'enable_to', 'event_name', 'event_start_datetime', 'event_end_datetime', 'event_location_name', 'event_track_name', 'event_abstract_list', 'event_badge_list', 'event_device_list', 'event_file_list', 'event_file_internal_use_list', 'event_location', 'event_location_list', 'event_person_list', 'event_presenter_cat', 'event_presentation_list', 'event_presenter_list', 'event_track', 'poc_event_person'}, - # 'table_name': 'v_event_session', - # 'base_name': Event_Session_Base, + # Legacy V2 keys: + 'table_name': 'v_event_session', + 'tbl_name_update': 'event_session', + 'base_name': Event_Session_Base, }, # Updated 2024-09-27 'archive': { @@ -277,6 +307,10 @@ obj_type_kv_li = { 'mdl_default': Archive_Base, 'mdl_in': Archive_Base, 'mdl_out': Archive_Base, + # Legacy V2 keys: + 'table_name': 'v_archive', + 'tbl_name_update': 'archive', + 'base_name': Archive_Base, 'exp_default': [ 'archive_id_random', 'account_id_random', @@ -304,6 +338,10 @@ obj_type_kv_li = { 'mdl_default': Archive_Content_Base, 'mdl_in': Archive_Content_Base, 'mdl_out': Archive_Content_Base, + # Legacy V2 keys: + 'table_name': 'v_archive_content', + 'tbl_name_update': 'archive_content', + 'base_name': Archive_Content_Base, 'exp_default': [ 'archive_content_id_random', 'archive_id_random', @@ -332,6 +370,10 @@ obj_type_kv_li = { 'mdl_default': Hosted_File_Base, 'mdl_in': Hosted_File_Base, 'mdl_out': Hosted_File_Base, + # Legacy V2 keys: + 'table_name': 'v_hosted_file', + 'tbl_name_update': 'hosted_file', + 'base_name': Hosted_File_Base, 'exp_default': [ 'hosted_file_id_random', 'hash_sha256', @@ -352,6 +394,10 @@ obj_type_kv_li = { 'mdl_default': Journal_Base, 'mdl_in': Journal_Base, 'mdl_out': Journal_Base, + # Legacy V2 keys: + 'table_name': 'v_journal', + 'tbl_name_update': 'journal', + 'base_name': Journal_Base, 'exp_default': [ 'journal_id_random', # 'account_id_random', @@ -369,6 +415,10 @@ obj_type_kv_li = { 'mdl_default': Journal_Entry_Base, 'mdl_in': Journal_Entry_Base, 'mdl_out': Journal_Entry_Base, + # Legacy V2 keys: + 'table_name': 'v_journal_entry', + 'tbl_name_update': 'journal_entry', + 'base_name': Journal_Entry_Base, 'exp_default': [ 'journal_entry_id_random', # 'account_id_random', 'journal_id_random', @@ -387,6 +437,11 @@ obj_type_kv_li = { 'mdl_default': Post_Base, 'mdl_in': Post_Base, 'mdl_out': Post_Base, + # Legacy V2 keys: + 'table_name': 'v_post', + 'table_name_alt': 'v_post_detail', + 'tbl_name_update': 'post', + 'base_name': Post_Base, 'exp_default': [ 'post_id_random', 'account_id_random', @@ -404,6 +459,11 @@ obj_type_kv_li = { 'mdl_default': Post_Comment_Base, 'mdl_in': Post_Comment_Base, 'mdl_out': Post_Comment_Base, + # Legacy V2 keys: + 'table_name': 'v_post_comment', + 'table_name_alt': 'v_post_comment_detail', + 'tbl_name_update': 'post_comment', + 'base_name': Post_Comment_Base, 'exp_default': [ 'post_comment_id_random', 'account_id_random', 'post_id_random', @@ -419,6 +479,10 @@ obj_type_kv_li = { 'mdl_default': Account_Base, 'mdl_in': Account_Base, 'mdl_out': Account_Base, + # Legacy V2 keys: + 'table_name': 'account', + 'tbl_name_update': 'account', + 'base_name': Account_Base, }, 'account_cfg': { 'tbl': 'account_cfg', @@ -428,6 +492,10 @@ obj_type_kv_li = { 'mdl_default': Account_Cfg_Base, 'mdl_in': Account_Cfg_Base, 'mdl_out': Account_Cfg_Base, + # Legacy V2 keys: + 'table_name': 'v_account_cfg', + 'tbl_name_update': 'account_cfg', + 'base_name': Account_Cfg_Base, }, 'address': { 'tbl': 'address', @@ -437,6 +505,10 @@ obj_type_kv_li = { 'mdl_default': Address_Base, 'mdl_in': Address_Base, 'mdl_out': Address_Base, + # Legacy V2 keys: + 'table_name': 'v_address', + 'tbl_name_update': 'address', + 'base_name': Address_Base, }, 'contact': { 'tbl': 'contact', @@ -446,6 +518,10 @@ obj_type_kv_li = { 'mdl_default': Contact_Base, 'mdl_in': Contact_Base, 'mdl_out': Contact_Base, + # Legacy V2 keys: + 'table_name': 'v_contact', + 'tbl_name_update': 'contact', + 'base_name': Contact_Base, }, 'data_store': { 'tbl': 'data_store', @@ -455,6 +531,10 @@ obj_type_kv_li = { 'mdl_default': Data_Store_Base, 'mdl_in': Data_Store_Base, 'mdl_out': Data_Store_Base, + # Legacy V2 keys: + 'table_name': 'v_data_store', + 'tbl_name_update': 'data_store', + 'base_name': Data_Store_Base, }, 'organization': { 'tbl': 'organization', @@ -464,6 +544,10 @@ obj_type_kv_li = { 'mdl_default': Organization_Base, 'mdl_in': Organization_Base, 'mdl_out': Organization_Base, + # Legacy V2 keys: + 'table_name': 'v_organization', + 'tbl_name_update': 'organization', + 'base_name': Organization_Base, }, 'page': { 'tbl': 'page', @@ -473,6 +557,10 @@ obj_type_kv_li = { 'mdl_default': Page_Base, 'mdl_in': Page_Base, 'mdl_out': Page_Base, + # Legacy V2 keys: + 'table_name': 'page', + 'tbl_name_update': 'page', + 'base_name': Page_Base, }, 'site': { 'tbl': 'site', @@ -482,6 +570,10 @@ obj_type_kv_li = { 'mdl_default': Site_Base, 'mdl_in': Site_Base, 'mdl_out': Site_Base, + # Legacy V2 keys: + 'table_name': 'site', + 'tbl_name_update': 'site', + 'base_name': Site_Base, }, 'site_domain': { 'tbl': 'site_domain', @@ -493,6 +585,12 @@ obj_type_kv_li = { 'mdl_alt': Site_Domain_FQDN_ID_Base, 'mdl_in': Site_Domain_Base, 'mdl_out': Site_Domain_Base, + # Legacy V2 keys: + 'table_name': 'v_site_domain', + 'table_name_alt': 'v_site_domain_fqdn_id', + 'tbl_name_update': 'site_domain', + 'base_name': Site_Domain_Base, + 'base_name_alt': Site_Domain_FQDN_ID_Base, }, 'order': { 'tbl': 'order', @@ -502,6 +600,10 @@ obj_type_kv_li = { 'mdl_default': Order_Base, 'mdl_in': Order_DB_Base, 'mdl_out': Order_Base, + # Legacy V2 keys: + 'table_name': 'v_order', + 'tbl_name_update': 'order', + 'base_name': Order_Base, }, 'order_line': { 'tbl': 'order_line', @@ -511,6 +613,10 @@ obj_type_kv_li = { 'mdl_default': Order_Line_Base, 'mdl_in': Order_Line_Base, 'mdl_out': Order_Line_Base, + # Legacy V2 keys: + 'table_name': 'v_order_line', + 'tbl_name_update': 'order_line', + 'base_name': Order_Line_Base, }, # Include the lookup lists. # Updated 2025-01-13 @@ -523,6 +629,10 @@ obj_type_kv_li = { 'mdl_default': None, 'mdl_in': None, 'mdl_out': None, + # Legacy V2 keys: + 'table_name': 'v_lu_country', + 'tbl_name_update': 'lu_country', + 'base_name': None, }, # Updated 2025-01-13 'lu_country_subdivision': { @@ -534,6 +644,10 @@ obj_type_kv_li = { 'mdl_default': None, 'mdl_in': None, 'mdl_out': None, + # Legacy V2 keys: + 'table_name': 'v_lu_country_subdivision', + 'tbl_name_update': 'lu_country_subdivision', + 'base_name': None, }, # Updated 2025-01-13 'lu_time_zone': { @@ -545,6 +659,10 @@ obj_type_kv_li = { 'mdl_default': None, 'mdl_in': None, 'mdl_out': None, + # Legacy V2 keys: + 'table_name': 'v_lu_time_zone', + 'tbl_name_update': 'lu_time_zone', + 'base_name': None, }, # Updated 2025-04-04 'person': { @@ -556,6 +674,10 @@ obj_type_kv_li = { 'mdl_default': Person_Base, 'mdl_in': Person_Base, # For input 'mdl_out': Person_Base, # For output + # Legacy V2 keys: + 'table_name': 'v_person', + 'tbl_name_update': 'person', + 'base_name': Person_Base, 'exp_default': [ 'person_id_random', 'given_name', 'middle_name', 'family_name', 'full_name', @@ -573,6 +695,10 @@ obj_type_kv_li = { 'mdl_default': User_Base, 'mdl_in': User_New_Base, # For input when creating a new user 'mdl_out': User_Out_Base, # For output + # Legacy V2 keys: + 'table_name': 'v_user', + 'tbl_name_update': 'user', + 'base_name': User_Base, 'exp_default': [ 'user_id_random', 'account_id_random', # Foreign key to account diff --git a/app/routers/api_crud_v3.py b/app/routers/api_crud_v3.py index 53e5422..682e193 100644 --- a/app/routers/api_crud_v3.py +++ b/app/routers/api_crud_v3.py @@ -27,6 +27,12 @@ async def health_check( ): """ Health check endpoint for V3 API. + + Architectural Choices: + - Non-blocking delay: Uses 'await asyncio.sleep' instead of 'time.sleep' to prevent + blocking the event loop, ensuring the Gunicorn worker can handle other requests. + - Granular Dependencies: Uses 'DelayParams' to handle optional latency simulation + consistently across all V3 endpoints via headers (X-Delay-ms) or query params (delay_ms). """ if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s) @@ -47,9 +53,12 @@ async def get_obj( ): """ Get a single top-level object by its random ID. - Examples: - - /v3/crud/journal/{journal_id} - - /v3/crud/account/{account_id} + + Special Cases: + - Object Resolution: Random IDs (id_random) are resolved to internal integer IDs + using Redis for performance, falling back to SQL if not found. + - Consistency: Uses 'obj_type_kv_li' from ae_obj_types_def.py to map URL paths + to database views/tables and Pydantic models. """ if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s) @@ -95,10 +104,13 @@ async def get_obj_li( ): """ Get a list of top-level objects. - Examples: - - /v3/crud/journal/ - - /v3/crud/journal/?for_obj_type=account&for_obj_id={account_id_random} - - /v3/crud/journal/?jp={"qry":[{"type":"AND","field":"for_type","operator":"=","value":"user"},{"type":"AND","field":"for_id","operator":"=","value":}]} + + Features: + - Status Filtering: Automatically filters by 'enabled' and 'hidden' status using + the StatusFilterParams dependency. + - Flexible Querying: Supports complex JSON-based queries via the 'jp' parameter. + - Contextual Filtering: Optionally filters by parent object relationship if + 'for_obj_type' and 'for_obj_id' are provided. """ if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s) @@ -213,8 +225,10 @@ async def post_obj( ): """ Create a new top-level object. - Examples: - - POST /v3/crud/journal/ (with Journal_Base in body) + + Validation: + - Uses 'mdl_in' from the object configuration to strictly validate incoming JSON data. + - 'data_to_insert' excludes unset fields to allow database defaults to apply. """ if delay.sleep_time_s > 0: await asyncio.sleep(delay.sleep_time_s) @@ -263,6 +277,539 @@ async def post_obj( return mk_resp(data=False, status_code=400, response=response, status_message="Failed to create object in database.") +@router.patch('/{obj_type_l1}/{obj_id}', response_model=Resp_Body_Base) +async def patch_obj( + request: Request, + response: Response, + obj_type_l1: str = Path(min_length=2, max_length=50), + obj_id: str = Path(min_length=11, max_length=22), + return_obj: Optional[bool] = True, + account: AccountContext = Depends(get_account_context), + serialization: SerializationParams = Depends(get_serialization_params), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Update a top-level object. + + Behavior: + - Partial Updates: Unlike POST, PATCH does not perform strict full-model validation, + allowing partial updates of only the fields provided in the body. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + obj_data = await request.json() + + obj_name = obj_type_l1 + if obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Object type '{obj_name}' not found.") + + obj_cfg = obj_type_kv_li[obj_name] + table_name_update = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + table_name_select = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + input_model = obj_cfg.get('mdl_in', obj_cfg.get('mdl')) + output_model = obj_cfg.get('mdl_out', obj_cfg.get('mdl_default', obj_cfg.get('mdl'))) + + if not table_name_update or not input_model or not table_name_select or not output_model: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' is incomplete.") + + record_id = redis_lookup_id_random(record_id_random=obj_id, table_name=obj_name) + if not record_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Object with ID '{obj_id}' not found.") + + # Validate incoming data with the appropriate Pydantic model. + # For PATCH, we don't want to fail on missing fields, so we don't validate like in POST. + # The sql_update function will only update the fields provided in the dict. + data_to_update = obj_data + + if sql_update_result := sql_update(data=data_to_update, table_name=table_name_update, record_id=record_id): + + if return_obj: + if sql_select_result := sql_select(table_name=table_name_select, record_id=record_id): + resp_data = output_model(**sql_select_result).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + return mk_resp(data=resp_data, response=response) + else: + return mk_resp(data=True, status_code=404, response=response, status_message="Object updated but could not be retrieved.") + else: + return mk_resp(data=True, response=response, status_message="Object updated successfully.") + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to update object in database. It may not have been found, or the data was invalid.") + + +@router.delete('/{obj_type_l1}/{obj_id}', response_model=Resp_Body_Base) +async def delete_obj( + response: Response, + obj_type_l1: str = Path(min_length=2, max_length=50), + obj_id: str = Path(min_length=11, max_length=22), + account: AccountContext = Depends(get_account_context), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Delete a top-level object. + + Soft Delete: + - Note that 'sql_delete' may implement soft-delete behavior depending on the + 'method' query parameter (delete, disable, hide), matching legacy behavior. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + obj_name = obj_type_l1 + if obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Object type '{obj_name}' not found.") + + obj_cfg = obj_type_kv_li[obj_name] + table_name_delete = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + + if not table_name_delete: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' is incomplete (missing table for deletion).") + + record_id = redis_lookup_id_random(record_id_random=obj_id, table_name=obj_name) + if not record_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Object with ID '{obj_id}' not found.") + + if sql_delete_result := sql_delete(table_name=table_name_delete, record_id=record_id): + return mk_resp(data=True, response=response, status_message=f"Object with ID '{obj_id}' deleted successfully.") + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to delete object in database. It may not have been found.") + + +@router.get('/{parent_obj_type}/{parent_obj_id}/{child_obj_type}/', response_model=Resp_Body_Base) +async def get_child_obj_li( + response: Response, + parent_obj_type: str, + parent_obj_id: str, + child_obj_type: str, + order_by_li: Optional[str] = None, + jp: Optional[Union[str, None]] = None, + account: AccountContext = Depends(get_account_context), + pagination: PaginationParams = Depends(get_pagination_params), + status_filter: StatusFilterParams = Depends(get_status_filter_params), + serialization: SerializationParams = Depends(get_serialization_params), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Get a list of child objects belonging to a parent. + + Nested URL Logic: + - This enforces parentage by using the parent's ID from the URL to filter the child list. + - Convention: Assumes the child table has a foreign key field named '{parent_obj_type}_id'. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + # This function's logic is very similar to get_obj_li, + # but it enforces the parent-child relationship from the URL path. + # We can treat the parent path parameters as if they were for_obj_type and for_obj_id query params. + + for_obj_type = parent_obj_type + for_obj_id = parent_obj_id + + qry_dict_li = None + fulltext_qry_dict_obj = None + and_qry_dict_obj = None + and_like_dict_obj = None + or_like_dict_obj = None + and_in_dict_li_obj = None + jp_obj = None + + if jp: + try: + jp_obj = json.loads(urllib.parse.unquote(jp)) + except Exception as e: + log.warning(e) + return mk_resp(data=False, status_code=400, response=response, status_message='The JSON string was not formatted correctly.') + + if jp_obj.get('qry'): + qry_dict_li = jp_obj['qry'] + if jp_obj.get('ft_qry'): + fulltext_qry_dict_obj = jp_obj['ft_qry'] + if jp_obj.get('and_qry'): + and_qry_dict_obj = jp_obj['and_qry'] + if jp_obj.get('and_like'): + and_like_dict_obj = jp_obj['and_like'] + if jp_obj.get('or_like'): + or_like_dict_obj = jp_obj['or_like'] + if jp_obj.get('and_in_li'): + and_in_dict_li_obj = jp_obj['and_in_li'] + + if order_by_li: + order_by_li = json.loads(order_by_li) + + obj_name = child_obj_type + if obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Object type '{obj_name}' not found.") + + obj_cfg = obj_type_kv_li[obj_name] + table_name = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + base_name = obj_cfg.get('mdl_default', obj_cfg.get('mdl')) + + if not table_name or not base_name: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' is incomplete.") + + # Resolve parent's random ID to integer ID + resolved_parent_id = redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type) + if not resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Parent object '{for_obj_type}' with ID '{for_obj_id}' not found.") + + field_name = f'{for_obj_type}_id' # Assuming convention like 'journal_id' + + sql_result = sql_select( + table_name=table_name, + field_name=field_name, + field_value=resolved_parent_id, + enabled=status_filter.enabled, + hidden=status_filter.hidden, + qry_dict_li=qry_dict_li, + fulltext_qry_dict=fulltext_qry_dict_obj, + and_qry_dict=and_qry_dict_obj, + and_like_dict=and_like_dict_obj, + or_like_dict=or_like_dict_obj, + and_in_dict_li=and_in_dict_li_obj, + order_by_li=order_by_li, + limit=pagination.limit, + offset=pagination.offset, + as_list=True, + ) + + if sql_result: + resp_data_li = [] + for record in sql_result: + resp_data = base_name(**record).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + resp_data_li.append(resp_data) + return mk_resp(data=resp_data_li, response=response) + else: + return mk_resp(data=[], status_code=200, response=response) # Return empty list on no results + + +@router.post('/{parent_obj_type}/{parent_obj_id}/{child_obj_type}/', response_model=Resp_Body_Base) +async def post_child_obj( + request: Request, + response: Response, + parent_obj_type: str = Path(min_length=2, max_length=50), + parent_obj_id: str = Path(min_length=11, max_length=22), + child_obj_type: str = Path(min_length=2, max_length=50), + return_obj: Optional[bool] = True, + account: AccountContext = Depends(get_account_context), + serialization: SerializationParams = Depends(get_serialization_params), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Create a new child object for a given parent. + + Logic: + - Auto-injection: Automatically resolves the parent's random ID and injects the + integer ID into the child's data before validation and insertion. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + obj_data = await request.json() + + parent_obj_name = parent_obj_type + child_obj_name = child_obj_type + + if parent_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Parent object type '{parent_obj_name}' not found.") + if child_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Child object type '{child_obj_name}' not found.") + + resolved_parent_id = redis_lookup_id_random(record_id_random=parent_obj_id, table_name=parent_obj_name) + if not resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Parent object '{parent_obj_name}' with ID '{parent_obj_id}' not found.") + + obj_cfg = obj_type_kv_li[child_obj_name] + table_name_insert = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + table_name_select = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + input_model = obj_cfg.get('mdl_in', obj_cfg.get('mdl')) + output_model = obj_cfg.get('mdl_out', obj_cfg.get('mdl_default', obj_cfg.get('mdl'))) + + if not table_name_insert or not input_model or not table_name_select or not output_model: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for child object type '{child_obj_name}' is incomplete.") + + # Inject the parent ID into the child object's data + parent_fk_field_name = f'{parent_obj_name}_id' + obj_data[parent_fk_field_name] = resolved_parent_id + + # Validate incoming data with the appropriate Pydantic model + try: + validated_obj = input_model(**obj_data) + except Exception as e: + log.warning(f"Validation error for {child_obj_name}: {e}") + return mk_resp(data=False, status_code=400, response=response, status_message=f"Validation error: {e}") + + # Convert to dict, excluding unset fields, for database insertion + data_to_insert = validated_obj.dict(exclude_unset=True) + + if sql_insert_result := sql_insert(data=data_to_insert, table_name=table_name_insert): + new_obj_id = sql_insert_result + new_obj_id_random = get_id_random(record_id=new_obj_id, table_name=child_obj_name) + + if return_obj: + if sql_select_result := sql_select(table_name=table_name_select, record_id=new_obj_id): + resp_data = output_model(**sql_select_result).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + return mk_resp(data=resp_data, response=response) + else: + return mk_resp(data={"obj_id": new_obj_id, "obj_id_random": new_obj_id_random}, status_code=404, response=response, status_message="Child object created but could not be retrieved.") + else: + return mk_resp(data={"obj_id": new_obj_id, "obj_id_random": new_obj_id_random}, response=response) + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to create child object in database.") + + +@router.get('/{parent_obj_type}/{parent_obj_id}/{child_obj_type}/{child_obj_id}', response_model=Resp_Body_Base) +async def get_child_obj( + response: Response, + parent_obj_type: str = Path(min_length=2, max_length=50), + parent_obj_id: str = Path(min_length=11, max_length=22), + child_obj_type: str = Path(min_length=2, max_length=50), + child_obj_id: str = Path(min_length=11, max_length=22), + account: AccountContext = Depends(get_account_context), + serialization: SerializationParams = Depends(get_serialization_params), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Get a single child object by its ID, ensuring it belongs to the correct parent. + + Security: + - Verifies that the child object's foreign key correctly points to the parent + provided in the URL, preventing access to unrelated child objects. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + parent_obj_name = parent_obj_type + child_obj_name = child_obj_type + + if parent_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Parent object type '{parent_obj_name}' not found.") + if child_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Child object type '{child_obj_name}' not found.") + + resolved_parent_id = redis_lookup_id_random(record_id_random=parent_obj_id, table_name=parent_obj_name) + if not resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Parent object '{parent_obj_name}' with ID '{parent_obj_id}' not found.") + + obj_cfg = obj_type_kv_li[child_obj_name] + table_name = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + base_name = obj_cfg.get('mdl_default', obj_cfg.get('mdl')) + + if not table_name or not base_name: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{child_obj_name}' is incomplete.") + + resolved_child_id = redis_lookup_id_random(record_id_random=child_obj_id, table_name=child_obj_name) + if not resolved_child_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object with ID '{child_obj_id}' not found.") + + if sql_result := sql_select(table_name=table_name, record_id=resolved_child_id): + # Verify the child belongs to the parent + parent_fk_field_name = f'{parent_obj_name}_id' + if sql_result.get(parent_fk_field_name) != resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_id}' not found under parent '{parent_obj_id}'.") + + resp_data = base_name(**sql_result).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + return mk_resp(data=resp_data, response=response) + else: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object with ID '{child_obj_id}' not found in database.") + + +@router.patch('/{parent_obj_type}/{parent_obj_id}/{child_obj_type}/{child_obj_id}', response_model=Resp_Body_Base) +async def patch_child_obj( + request: Request, + response: Response, + parent_obj_type: str = Path(min_length=2, max_length=50), + parent_obj_id: str = Path(min_length=11, max_length=22), + child_obj_type: str = Path(min_length=2, max_length=50), + child_obj_id: str = Path(min_length=11, max_length=22), + return_obj: Optional[bool] = True, + account: AccountContext = Depends(get_account_context), + serialization: SerializationParams = Depends(get_serialization_params), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Update a child object by its ID, ensuring it belongs to the correct parent. + + Verification: + - Like GET, PATCH verifies parentage before applying any updates to ensure data integrity. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + obj_data = await request.json() + + parent_obj_name = parent_obj_type + child_obj_name = child_obj_type + + if parent_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Parent object type '{parent_obj_name}' not found.") + if child_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Child object type '{child_obj_name}' not found.") + + # Resolve IDs + resolved_parent_id = redis_lookup_id_random(record_id_random=parent_obj_id, table_name=parent_obj_name) + if not resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Parent object '{parent_obj_name}' with ID '{parent_obj_id}' not found.") + + resolved_child_id = redis_lookup_id_random(record_id_random=child_obj_id, table_name=child_obj_name) + if not resolved_child_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_name}' with ID '{child_obj_id}' not found.") + + # Get config for child object + obj_cfg = obj_type_kv_li[child_obj_name] + table_name_update = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + table_name_select = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + output_model = obj_cfg.get('mdl_out', obj_cfg.get('mdl_default', obj_cfg.get('mdl'))) + + if not table_name_update or not table_name_select or not output_model: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for child object type '{child_obj_name}' is incomplete.") + + # Verify parentage before updating + if existing_child := sql_select(table_name=table_name_select, record_id=resolved_child_id): + parent_fk_field_name = f'{parent_obj_name}_id' + if existing_child.get(parent_fk_field_name) != resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_id}' not found under parent '{parent_obj_id}'.") + else: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_id}' not found.") + + # The sql_update function will only update the fields provided in the dict. + data_to_update = obj_data + + if sql_update(data=data_to_update, table_name=table_name_update, record_id=resolved_child_id): + if return_obj: + if updated_child := sql_select(table_name=table_name_select, record_id=resolved_child_id): + resp_data = output_model(**updated_child).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + return mk_resp(data=resp_data, response=response) + else: + return mk_resp(data=True, status_code=404, response=response, status_message="Object updated but could not be retrieved post-update.") + else: + return mk_resp(data=True, response=response, status_message="Object updated successfully.") + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to update object in database.") + + +@router.delete('/{parent_obj_type}/{parent_obj_id}/{child_obj_type}/{child_obj_id}', response_model=Resp_Body_Base) +async def delete_child_obj( + response: Response, + parent_obj_type: str = Path(min_length=2, max_length=50), + parent_obj_id: str = Path(min_length=11, max_length=22), + child_obj_type: str = Path(min_length=2, max_length=50), + child_obj_id: str = Path(min_length=11, max_length=22), + account: AccountContext = Depends(get_account_context), + delay: DelayParams = Depends(get_delay_params), + ): + """ + Delete a child object by its ID, ensuring it belongs to the correct parent. + + Safety: + - Enforces parentage verification before deletion to prevent unauthorized data removal. + """ + if delay.sleep_time_s > 0: + await asyncio.sleep(delay.sleep_time_s) + + log.setLevel(logging.WARNING) + log.debug(locals()) + + parent_obj_name = parent_obj_type + child_obj_name = child_obj_type + + if parent_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Parent object type '{parent_obj_name}' not found.") + if child_obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Child object type '{child_obj_name}' not found.") + + # Resolve IDs + resolved_parent_id = redis_lookup_id_random(record_id_random=parent_obj_id, table_name=parent_obj_name) + if not resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Parent object '{parent_obj_name}' with ID '{parent_obj_id}' not found.") + + resolved_child_id = redis_lookup_id_random(record_id_random=child_obj_id, table_name=child_obj_name) + if not resolved_child_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_name}' with ID '{child_obj_id}' not found.") + + # Get config for child object + obj_cfg = obj_type_kv_li[child_obj_name] + table_name_delete = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + table_name_select = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) # For verification + + if not table_name_delete or not table_name_select: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for child object type '{child_obj_name}' is incomplete.") + + # Verify parentage before deleting + if existing_child := sql_select(table_name=table_name_select, record_id=resolved_child_id): + parent_fk_field_name = f'{parent_obj_name}_id' + if existing_child.get(parent_fk_field_name) != resolved_parent_id: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_id}' not found under parent '{parent_obj_id}'.") + else: + return mk_resp(data=False, status_code=404, response=response, status_message=f"Child object '{child_obj_id}' not found.") + + # If verification passes, delete the object + if sql_delete(table_name=table_name_delete, record_id=resolved_child_id): + return mk_resp(data=True, response=response, status_message=f"Object with ID '{child_obj_id}' deleted successfully.") + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to delete object in database.") + + log.setLevel(logging.WARNING) + log.debug(locals()) + + obj_data = await request.json() + + obj_name = obj_type_l1 + if obj_name not in obj_type_kv_li: + return mk_resp(data=False, status_code=400, response=response, status_message=f"Object type '{obj_name}' not found.") + + obj_cfg = obj_type_kv_li[obj_name] + table_name_insert = obj_cfg.get('tbl_update', obj_cfg.get('tbl')) + table_name_select = obj_cfg.get('tbl_default', obj_cfg.get('tbl')) + input_model = obj_cfg.get('mdl_in', obj_cfg.get('mdl')) + output_model = obj_cfg.get('mdl_out', obj_cfg.get('mdl_default', obj_cfg.get('mdl'))) + + if not table_name_insert or not input_model or not table_name_select or not output_model: + return mk_resp(data=False, status_code=500, response=response, status_message=f"Configuration for object type '{obj_name}' is incomplete.") + + # Validate incoming data with the appropriate Pydantic model + try: + validated_obj = input_model(**obj_data) + except Exception as e: + log.warning(f"Validation error for {obj_name}: {e}") + return mk_resp(data=False, status_code=400, response=response, status_message=f"Validation error: {e}") + + # Convert to dict, excluding unset fields, for database insertion + data_to_insert = validated_obj.dict(exclude_unset=True) + + if sql_insert_result := sql_insert(data=data_to_insert, table_name=table_name_insert): + new_obj_id = sql_insert_result + new_obj_id_random = get_id_random(record_id=new_obj_id, table_name=obj_name) + + if return_obj: + if sql_select_result := sql_select(table_name=table_name_select, record_id=new_obj_id): + resp_data = output_model(**sql_select_result).dict(by_alias=serialization.by_alias, exclude_unset=serialization.exclude_unset) + return mk_resp(data=resp_data, response=response) + else: + return mk_resp(data={"obj_id": new_obj_id, "obj_id_random": new_obj_id_random}, status_code=404, response=response, status_message="Object created but could not be retrieved.") + else: + return mk_resp(data={"obj_id": new_obj_id, "obj_id_random": new_obj_id_random}, response=response) + else: + return mk_resp(data=False, status_code=400, response=response, status_message="Failed to create object in database.") + + @router.patch('/{obj_type_l1}/{obj_id}', response_model=Resp_Body_Base) async def patch_obj( request: Request, diff --git a/documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md b/documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md new file mode 100644 index 0000000..deb39d8 --- /dev/null +++ b/documentation/V3_CRUD_ARCHITECTURE_AND_LEARNINGS.md @@ -0,0 +1,53 @@ +# Aether API V3 CRUD: Architecture and Learnings + +This document summarizes the development of the V3 CRUD API, the architectural choices made, and the lessons learned during the process. + +## 1. V3 CRUD Architecture + +The V3 CRUD API (`/v3/crud/`) is designed to run in parallel with legacy V1 and V2 endpoints. It introduces a hierarchical, nested URL structure and leverages modern FastAPI features for better maintainability and performance. + +### Key Features: +- **Nested URL Structure**: Enforces parent-child relationships (e.g., `/v3/crud/site/{site_id}/site_domain/`). +- **Granular Dependencies**: Instead of a monolithic common parameters object, V3 uses specialized, reusable dependencies from `app/lib_general_v3.py`: + - `AccountContext`: Resolves account ID with clear precedence (Header > Query Token > Bypass Header). + - `PaginationParams`: Standardizes `limit` and `offset`. + - `StatusFilterParams`: Handles `enabled` and `hidden` status filtering. + - `SerializationParams`: Controls Pydantic serialization options (`by_alias`, `exclude_unset`). + - `DelayParams`: Facilitates optional latency simulation for testing. +- **Non-blocking Delay**: Uses `await asyncio.sleep()` to simulate network latency without blocking the Gunicorn worker's event loop. +- **Data-Driven Configuration**: Uses the modern format in `app/ae_obj_types_def.py` to map objects to tables and models. + +## 2. Backward Compatibility Strategy + +To ensure that the introduction of V3 doesn't break legacy V1 and V2 endpoints: +- **Parallel Routes**: All V3 logic is isolated in `app/routers/api_crud_v3.py`. +- **Hybrid Configuration**: The `obj_type_kv_li` dictionary in `app/ae_obj_types_def.py` has been updated to include both modern keys (e.g., `tbl`, `mdl`) and legacy keys (e.g., `table_name`, `base_name`). This allows V2 endpoints to continue functioning normally while V3 endpoints use the refined structure. +- **Stable Core Imports**: Core modules like `app/lib_general.py` and `app/models/response_models.py` have been refactored to maintain their existing exports (`log`, `logging`, `mk_resp`) while internally adopting more robust module-level logging. + +## 3. Learnings and Best Practices + +### Logging and Startup Stability +- **Isolate Loggers**: Modules should instantiate their own module-level loggers (`logging.getLogger(__name__)`) rather than importing a global instance. This breaks circular dependencies and improves traceability. +- **Robust Configuration**: Logging configuration (`dictConfig`) should be wrapped in `try...except` to prevent application crashes due to environment issues (e.g., missing log directories in Docker). +- **Explicit Imports**: Always use `import logging.config` before calling `logging.config.dictConfig` to ensure the module is fully initialized. + +### FastAPI and Pydantic +- **Dependency Injection**: Use `response: Response` as a type hint for standard injection. Avoid `Depends(Response)` as it is not a valid dependency provider and can cause router initialization failures. +- **Python Parameter Order**: In function signatures, non-default arguments (like `response: Response`) must precede arguments with default values or `Depends()`. +- **Async Concurrency**: Use `asyncio.sleep()` instead of `time.sleep()` in async endpoints. Blocking the event loop in a high-concurrency environment leads to worker timeouts and `502 Bad Gateway` errors. +- **Pydantic Compatibility**: Ensure all new models and utility functions remain compatible with Pydantic v1.10 until a full project-wide migration to v2 is planned. Avoid V2-only features like `computed_field` or `model_validator`. + +## 4. Current Migration Status + +The following objects have been migrated to the modern V3 configuration and are supported by the V3 API: +- `journal`, `journal_entry` +- `site`, `site_domain` +- `account`, `account_cfg` +- `address`, `contact` +- `order`, `order_line` +- `organization`, `page` +- `data_store`, `activity_log` +- `archive`, `archive_content` +- `hosted_file`, `post`, `post_comment` +- `person`, `user` +- `lu_country`, `lu_country_subdivision`, `lu_time_zone`