From a625d33995d2be39e6c6453feeb3e072acc70f52 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 7 Sep 2021 14:22:57 -0400 Subject: [PATCH] General code clean up for SQL functions and related. --- app/db_sql.py | 227 +++++++++++++++++++++++++++++--------------------- 1 file changed, 134 insertions(+), 93 deletions(-) diff --git a/app/db_sql.py b/app/db_sql.py index ed2d66d..01971b6 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -25,7 +25,14 @@ db = engine.connect() # ### BEGIN ### Core Help CRUD ### sql_insert() ### -def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, rm_id_random:bool=False, id_random_length:int=8): +# Updated 2021-09-07 +def sql_insert( + sql: str|None = None, + data: dict|None = None, + table_name: str|None = None, + rm_id_random: bool = False, + id_random_length: int = 8, + ): #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -76,39 +83,39 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, # http://sqlalche.me/e/14/gkpj # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" trans.rollback() - log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^') - log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****') log.error('**** *** ** * SQL Statement: * ** *** ****') log.error(e.statement) log.error('**** *** ** * SQL Parameters: * ** *** ****') log.error(e.params) log.error('**** *** ** * SQL Origin Message: * ** *** ****') log.error(e.orig) - log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****') return None except OperationalError as e: # Likely an unknown field or related trans.rollback() - log.error('This is likely because there was an unknown field or similar included. Returning False') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^') - log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****') log.error('**** *** ** * SQL Statement: * ** *** ****') log.error(e.statement) log.error('**** *** ** * SQL Parameters: * ** *** ****') log.error(e.params) log.error('**** *** ** * SQL Origin Message: * ** *** ****') log.error(e.orig) - log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****') return False except Exception as e: trans.rollback() - log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') + log.error('An unknown exception happened. Returning False') log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') @@ -121,24 +128,6 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, log.error(e.orig) log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') - # print('**** *** ** * ### ### * ** *** ****') - # log.error(e.code) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.statement) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.params) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.orig) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.ismulti) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.hide_parameters) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.detail) - # log.error('**** *** ** * ### ### * ** *** ****') - # log.error(e.connection_invalidated) - # log.error('**** *** ** * ### ### * ** *** ****') - return False else: log.debug(result_insert) @@ -167,7 +156,16 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, # ### BEGIN ### Core Help CRUD ### sql_update() ### -def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, record_id:int|None=None, record_id_random:str|None=None, rm_id_random:bool=False, id_random_length:None|int=None): +# Updated 2021-09-07 +def sql_update( + sql: str|None = None, + data: dict|None = None, + table_name: str|None = None, + record_id: int|None = None, + record_id_random: str|None = None, + rm_id_random: bool = False, + id_random_length: None|int = None + ): #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -220,23 +218,39 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, # http://sqlalche.me/e/14/gkpj # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" trans.rollback() - log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^') - log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****') log.error('**** *** ** * SQL Statement: * ** *** ****') log.error(e.statement) log.error('**** *** ** * SQL Parameters: * ** *** ****') log.error(e.params) log.error('**** *** ** * SQL Origin Message: * ** *** ****') log.error(e.orig) - log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****') return None except OperationalError as e: # Likely an unknown field or related trans.rollback() - log.error('This is likely because there was an unknown field or similar included. Returning False') + log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****') + + return False + except Exception as e: + trans.rollback() + log.error('An unknown exception happened. Returning False') log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') @@ -249,11 +263,6 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, log.error(e.orig) log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') - return False - except Exception as e: - trans.rollback() - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False else: log.debug(result_update) @@ -280,12 +289,13 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, # ### BEGIN ### Core Help CRUD ### sql_insert_or_update() ### # The catch all SQL INSERT or UPDATE function - STI 2021-02-17 # This one does it all for SQL INSERT and UPDATE queries +# Updated 2021-09-07 def sql_insert_or_update( sql: str|None = None, data: dict|None = None, table_name: str|None = None, rm_id_random: bool = False, - id_random_length: int|None = None + id_random_length: int|None = None, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -343,40 +353,49 @@ def sql_insert_or_update( # http://sqlalche.me/e/14/gkpj # Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'" trans.rollback() - log.error('This is likely because there was an attempt to create a duplicate entry. Returning None') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None') + log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^') - log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****') log.error('**** *** ** * SQL Statement: * ** *** ****') log.error(e.statement) log.error('**** *** ** * SQL Parameters: * ** *** ****') log.error(e.params) log.error('**** *** ** * SQL Origin Message: * ** *** ****') log.error(e.orig) - log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****') return None except OperationalError as e: # Likely an unknown field or related trans.rollback() - log.error('This is likely because there was an unknown field or similar included. Returning False') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False') + log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^') - log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****') log.error('**** *** ** * SQL Statement: * ** *** ****') log.error(e.statement) log.error('**** *** ** * SQL Parameters: * ** *** ****') log.error(e.params) log.error('**** *** ** * SQL Origin Message: * ** *** ****') log.error(e.orig) - log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****') return False except Exception as e: trans.rollback() log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') return False else: log.debug(result_insert_or_update) @@ -406,6 +425,7 @@ def sql_insert_or_update( # ### BEGIN ### Core Help CRUD ### sql_select() ### # The catch all SQL SELECT function - STI 2021-02-17 # This one does it all for SQL SELECT queries +# Updated 2021-09-07 def sql_select( table_name: str|None = None, record_id: int|None = None, @@ -417,7 +437,7 @@ def sql_select( rm_id_random: bool = False, as_dict: bool|None = True, as_list: bool|None = False, - max_count: int = 100000 + max_count: int = 100000, ): current_log_level = log.level log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL @@ -536,14 +556,11 @@ def sql_select( log.info('Executing with SQL statement only...') result = db.execute(sql) except OperationalError as e: - log.warning('*** An exception happened: OperationalError ***') - log.warning('* This is likely a "MySQL server has gone away" error. Going to try again... *') - log.warning(repr(e)) - log.warning('***') - log.warning(str(e)) - log.warning('^^^ exception ^^^') + log.error('An operational error exception happended. This is likely a "MySQL server has gone away" error. Going to try again...') + log.exception('**** *** ** * ### BEGIN ### Operational Exception Happened: Trying again... * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Exception ^ ^^ ^^^ ^^^^') - log.warning('Trying to recreate the pool...') + log.info('Trying to recreate the pool...') log.debug('############## ############') log.debug(dir(db)) log.debug(vars(db)) @@ -555,15 +572,16 @@ def sql_select( log.debug(vars(db.engine.pool)) log.debug('############## ############') db.engine.dispose() - log.warning('Now trying the query again...') + log.info('Now trying the query again...') try: if data: - log.warning('2x Executing with SQL statement and data...') + log.info('2x Executing with SQL statement and data...') result = db.execute(sql, data) else: - log.warning('2x Executing with SQL statement only...') + log.info('2x Executing with SQL statement only...') result = db.execute(sql) except Exception as e: + log.error('Tried again an exception was raised again. Not going to try again.') log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful @@ -571,9 +589,19 @@ def sql_select( log.info('Successfully executed the SQL on the second try.') pass except Exception as e: - log.info('An exception happened. Returning False.') + log.error('An unknown exception happened. Returning False.') log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + return False # Not successful else: log.info('Successfully executed the SQL on the first try.') @@ -637,14 +665,15 @@ def sql_select( # ### BEGIN ### Core Help CRUD ### sql_delete() ### # The catch all SQL DELETE function - STI 2021-02-17 # This one does it all for SQL DELETE queries +# Updated 2021-09-07 def sql_delete( - table_name:str|None=None, - record_id:int|None=None, - record_id_random:str|None=None, - field_name:str|None=None, - field_value=None, - sql:str|None=None, - data:dict|None=None + table_name: str|None = None, + record_id: int|None = None, + record_id_random: str|None = None, + field_name: str|None = None, + field_value = None, + sql: str|None = None, + data: dict|None = None ): # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -728,15 +757,11 @@ def sql_delete( log.debug(dir(result)) log.debug(vars(result)) except OperationalError as e: - log.warning('This is likely a "MySQL server has gone away" error. Going to try again...') - log.exception('**** *** ** * ### BEGIN ### Exception Happened: OperationalError MySQL server connection? * ** *** ****') - log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') - # log.warning(repr(e)) - # log.warning('***') - # log.warning(str(e)) - # log.warning('^^^ exception ^^^') + log.error('An operational error exception happended. This is likely a "MySQL server has gone away" error. Going to try again...') + log.exception('**** *** ** * ### BEGIN ### Operational Exception Happened: Trying again... * ** *** ****') + log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Exception ^ ^^ ^^^ ^^^^') - log.warning('Trying to recreate the pool...') + log.info('Trying to recreate the pool...') log.debug('############## ############') log.debug(dir(db)) log.debug(vars(db)) @@ -748,27 +773,38 @@ def sql_delete( log.debug(vars(db.engine.pool)) log.debug('############## ############') db.engine.dispose() - log.warning('Now trying the query again...') + log.info('Now trying the query again...') try: log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL if data: - log.warning('2x Executing with SQL statement and data...') + log.info('2x Executing with SQL statement and data...') result = db.execute(sql, data) else: - log.warning('2x Executing with SQL statement only...') + log.info('2x Executing with SQL statement only...') result = db.execute(sql) log.debug(result) except Exception as e: - log.warning('Tried again an exception was raised again. Not going to try again.') - log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: * ** *** ****') + log.error('Tried again an exception was raised again. Not going to try again.') + log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') return False # Not successful else: log.info('Successfully executed the SQL on the second try.') pass except Exception as e: + log.error('An unknown exception happened. Returning False.') log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****') log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^') + + log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****') + log.error('**** *** ** * SQL Statement: * ** *** ****') + log.error(e.statement) + log.error('**** *** ** * SQL Parameters: * ** *** ****') + log.error(e.params) + log.error('**** *** ** * SQL Origin Message: * ** *** ****') + log.error(e.orig) + log.error('**** *** ** * ### END ### Exception Details: * ** *** ****') + return False # Not successful else: log.info('Successfully executed the SQL on the first try.') @@ -776,9 +812,7 @@ def sql_delete( # NOTE: Need to deal with 0 rows affected when the WHERE clause was not satisfied and there was no error. return True # Successful - - - +# ### END ### Core Help CRUD ### sql_delete() ### # ### BEGIN ### API DB SQL ### redis_lookup_id_random() ### @@ -788,7 +822,10 @@ def sql_delete( # If success then return the ID number # If not success and there is a table_name then check the database table passed # If found in database table then store in Redis and return the ID number -def redis_lookup_id_random(record_id_random:int|str, table_name:str): +def redis_lookup_id_random( + record_id_random: int|str, + table_name: str, + ): current_log_level = log.level log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -873,7 +910,10 @@ def redis_lookup_id_random(record_id_random:int|str, table_name:str): # ### BEGIN ### API DB SQL ### get_id_random() ### # Changed name from lookup_id_random() to get_id_random() # Updated 2021-08-23 -def get_id_random(record_id:int, table_name:str): +def get_id_random( + record_id: int, + table_name: str + ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -912,7 +952,9 @@ def get_id_random(record_id:int, table_name:str): # Look up and resolve id_random values to their id # Remove the unneeded *_id_random key from the dict # This really needs to be simplified... Use a list of dicts instead. Can store as JSON in the DB. -def lookup_id_random_pop(obj_data:dict): +def lookup_id_random_pop( + obj_data: dict + ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -1138,7 +1180,6 @@ def lookup_id_random_pop(obj_data:dict): # ### END ### API DB SQL ### lookup_id_random_pop() ### - # ### BEGIN ### API DB SQL Methods ### get_account_id_w_for_type_id() ### # Updated 2021-08-24 def get_account_id_w_for_type_id( @@ -1169,4 +1210,4 @@ def get_account_id_w_for_type_id( if account_id := for_data_result.get('account_id', None): return account_id else: return False else: return None -# ### END ### API DB SQL Methods ### get_account_id_w_for_type_id() ### \ No newline at end of file +# ### END ### API DB SQL Methods ### get_account_id_w_for_type_id() ###