diff --git a/app/db_sql.py b/app/db_sql.py index f62729e..ed2d66d 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -72,15 +72,73 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, try: result_insert = db.execute(sql_insert, data) trans.commit() - except Exception as e: + except IntegrityError as e: # Specifically want to capture duplicate entry attempts # http://sqlalche.me/e/14/gkpj # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" trans.rollback() - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return None + except OperationalError as e: # Likely an unknown field or related + trans.rollback() + log.error('This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return False + except Exception as e: + trans.rollback() + log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + # print('**** *** ** * ### ### * ** *** ****') + # log.error(e.code) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.statement) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.params) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.orig) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.ismulti) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.hide_parameters) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.detail) + # log.error('**** *** ** * ### ### * ** *** ****') + # log.error(e.connection_invalidated) + # log.error('**** *** ** * ### ### * ** *** ****') + return False else: log.debug(result_insert) @@ -158,13 +216,44 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, log.info('Trying to execute the SQL UPDATE query...') result_update = db.execute(sql_update, data) trans.commit() + except IntegrityError as e: # Specifically want to capture duplicate entry attempts + # http://sqlalche.me/e/14/gkpj + # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" + trans.rollback() + log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return None + except OperationalError as e: # Likely an unknown field or related + trans.rollback() + log.error('This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return False except Exception as e: trans.rollback() - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False else: log.debug(result_update) @@ -250,13 +339,44 @@ def sql_insert_or_update( log.debug(data) result_insert_or_update = db.execute(sql_insert_or_update, data) trans.commit() + except IntegrityError as e: # Specifically want to capture duplicate entry attempts + # http://sqlalche.me/e/14/gkpj + # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" + trans.rollback() + log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return None + except OperationalError as e: # Likely an unknown field or related + trans.rollback() + log.error('This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + + return False except Exception as e: trans.rollback() - log.exception('*** An exception happened. ***') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False else: log.debug(result_insert_or_update) @@ -444,21 +564,16 @@ def sql_select( log.warning('2x Executing with SQL statement only...') result = db.execute(sql) except Exception as e: - log.warning('2x A *second* exception happened. Returning False.') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful else: log.info('Successfully executed the SQL on the second try.') pass except Exception as e: log.info('An exception happened. Returning False.') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful else: log.info('Successfully executed the SQL on the first try.') @@ -613,12 +728,13 @@ def sql_delete( log.debug(dir(result)) log.debug(vars(result)) except OperationalError as e: - log.warning('*** An exception happened: OperationalError ***') - log.warning('* This is likely a "MySQL server has gone away" error. Going to try again... *') - log.warning(repr(e)) - log.warning('***') - log.warning(str(e)) - log.warning('^^^ exception ^^^') + log.warning('This is likely a "MySQL server has gone away" error. Going to try again...') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: OperationalError MySQL server connection? * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + # log.warning(repr(e)) + # log.warning('***') + # log.warning(str(e)) + # log.warning('^^^ exception ^^^') log.warning('Trying to recreate the pool...') log.debug('############## ############') @@ -643,21 +759,16 @@ def sql_delete( result = db.execute(sql) log.debug(result) except Exception as e: - log.warning('2x A *second* exception happened. Returning False.') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.warning('Tried again an exception was raised again. Not going to try again.') + log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful else: log.info('Successfully executed the SQL on the second try.') pass except Exception as e: - log.info('An exception happened. Returning False.') - log.exception(repr(e)) - log.exception('***') - log.exception(str(e)) - log.exception('^^^ exception ^^^') + log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful else: log.info('Successfully executed the SQL on the first try.') diff --git a/app/methods/hosted_file_methods.py b/app/methods/hosted_file_methods.py index dbcb20c..4a4b025 100644 --- a/app/methods/hosted_file_methods.py +++ b/app/methods/hosted_file_methods.py @@ -218,7 +218,7 @@ def create_hosted_file_link( link_to_type: str, link_to_id: int|str, ): - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass @@ -234,8 +234,15 @@ def create_hosted_file_link( hosted_file_link_data['link_to_type'] = link_to_type # Should this be renamed to "link_to_type" for clarity? hosted_file_link_data['link_to_id'] = link_to_id # Should this be renamed to "link_to_id" for clarity? - # NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. - if hosted_file_link_data_in_result := sql_insert(data=hosted_file_link_data, table_name='hosted_file_link', id_random_length=0): pass # This should be improved + hosted_file_link_data['test'] = 'test' + + # NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID. + if hosted_file_link_data_in_result := sql_insert(data=hosted_file_link_data, table_name='hosted_file_link', id_random_length=0): + log.info('The hosted_file_link was created.') + pass # This should be improved + elif hosted_file_link_data_in_result is None: + log.info('The hosted_file_link probably already exists.') + return None else: # This should be improved log.warning('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.') diff --git a/app/models/hosted_file_models.py b/app/models/hosted_file_models.py index b6c619e..0a909d6 100644 --- a/app/models/hosted_file_models.py +++ b/app/models/hosted_file_models.py @@ -32,7 +32,7 @@ class Hosted_File_Base(BaseModel): version: Optional[int] directory_path: Optional[str] - subdirectory_path: Optional[str] + subdirectory_path: Optional[str] # NOTE: This will frequently only contain numbers, but it still needs to be a string filename: Optional[str] extension: Optional[str] content_type: Optional[str] diff --git a/app/routers/event_session.py b/app/routers/event_session.py index 4f45785..5ca38a7 100644 --- a/app/routers/event_session.py +++ b/app/routers/event_session.py @@ -70,7 +70,7 @@ async def post_event_session_obj_new_v4( return mk_resp(data=data, response=response, status_message='The event session was created.') else: return mk_resp(data=False, status_code=400, response=response, status_message='The result from trying to create an event session was unexpected.') -# ### BEGIN ### API Event Session ### post_event_session_obj_new_v4() ### +# ### END ### API Event Session ### post_event_session_obj_new_v4() ### # ### BEGIN ### API Event Session ### patch_event_session_obj_exist_v4() ### diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index 7c84da8..c1414c2 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -185,6 +185,10 @@ async def upload_files_fake( hosted_file_list = [] for file_info in file_info_li: + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(file_info) + + # "saved" means that the file was or is now saved on the file server if file_info['saved']: # Create a new host_file object entry log.info('Check and create a new host_file object entry...') @@ -199,7 +203,8 @@ async def upload_files_fake( field_value = file_info['hash_sha256'], ): # NOTE: Since the file already exists and something was in the database, it may need to be updated with the new subdirectory_path. - hosted_file_id = hosted_file_sel_result.get('id_random', None) + hosted_file_id = hosted_file_sel_result.get('id', None) + hosted_file_id_random = hosted_file_sel_result.get('id_random', None) hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) # ****************************************************** @@ -208,21 +213,49 @@ async def upload_files_fake( # NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory. log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - if file_info['already_exists'] and not file_info['already_exists_subdir']: # This means the database record probably needs to be updated with the new subdirectory_path field. - subdirectory_path_from_passed_data = file_info['subdirectory_path'] - log.info(f'The database record probably needs to be updated with the new subdirectory_path field. Subdirectory Path (from passed data): {subdirectory_path_from_passed_data}') + log.debug(file_info['already_exists']) + log.debug(file_info['already_exists_subdir']) + log.debug(file_info['subdirectory_path']) + # if not hosted_file_dict.get('subdirectory_path', None): # This means the database record probably needs to be updated with the new subdirectory_path field. + # subdirectory_path = file_info['subdirectory_path'] + # log.info(f'The database record probably needs to be updated with the new subdirectory_path field. Subdirectory Path (from passed data): {subdirectory_path}') + + # hosted_file_data_up = {} + # hosted_file_data_up['id'] = hosted_file_id + # hosted_file_data_up['subdirectory_path'] = subdirectory_path + + # if hosted_file_up_result := sql_update( + # table_name = 'hosted_file', + # data = hosted_file_data_up, + # ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + # else: + # log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + # log.debug(hosted_file_up_result) if subdirectory_path := hosted_file_dict.get('subdirectory_path'): - log.info(f'The new subdirectory_path field was found? Subdirectory Path: {subdirectory_path}') - else: - log.info(f'The new subdirectory_path field was not found? This needs to be updated. Subdirectory Path: {subdirectory_path}') + log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}') + elif subdirectory_path := file_info.get('subdirectory_path', None): + log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}') + hosted_file_data_up = {} + hosted_file_data_up['id'] = hosted_file_id + hosted_file_data_up['subdirectory_path'] = subdirectory_path + + if hosted_file_up_result := sql_update( + table_name = 'hosted_file', + data = hosted_file_data_up, + ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + else: + log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + log.debug(hosted_file_up_result) + else: + log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.') # ****************************************************** # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(hosted_file_dict) else: - # SOMETHING WENT WRONG + # NOTE: SOMETHING WENT WRONG # Going to try and create a new host_file entry... log.warning('For some reason a host_file object entry with the has was not found.') # file_info['id_random'] = None @@ -248,8 +281,42 @@ async def upload_files_fake( # Got existing host_file object_entry! # Odd... the hash was found in the database, but the file had to be copied again. # If this happens then the file on the host server was probably deleted at some point. - hosted_file_id = hosted_file_sel_result.get('id_random', None) + hosted_file_id = hosted_file_sel_result.get('id', None) + hosted_file_id_random = hosted_file_sel_result.get('id_random', None) hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) + + # ****************************************************** + # New as of 2021-08-26 + + # NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory. + + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(file_info['already_exists']) + log.debug(file_info['already_exists_subdir']) + log.debug(file_info['subdirectory_path']) + + if subdirectory_path := hosted_file_dict.get('subdirectory_path'): + log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}') + elif subdirectory_path := file_info.get('subdirectory_path', None): + log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}') + + hosted_file_data_up = {} + hosted_file_data_up['id'] = hosted_file_id + hosted_file_data_up['subdirectory_path'] = subdirectory_path + + if hosted_file_up_result := sql_update( + table_name = 'hosted_file', + data = hosted_file_data_up, + ): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + else: + log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}') + log.debug(hosted_file_up_result) + else: + log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.') + # ****************************************************** + + # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(hosted_file_dict) else: # This is normal since the file was not found on the host server and not found in the DB. # Create a new host_file object entry and new host_file.id_random. @@ -264,7 +331,7 @@ async def upload_files_fake( hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member log.debug(hosted_file_obj_result) log.debug(hosted_file_sel_result) - else: + else: # The file was not and is not saved on the file server file_info['id_random'] = None hosted_file_obj = Hosted_File_Base(**file_info) hosted_file_id = None