Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

General code clean up for SQL functions and related.

Browse Source
This commit is contained in:
Scott Idem
2021-09-07 14:22:57 -04:00
parent af0da563f1
commit a625d33995
1 changed files with 134 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
app/db_sql.py
View File

@@ -25,7 +25,14 @@ db = engine.connect()
# ### BEGIN ### Core Help CRUD ### sql_insert() ###
def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, rm_id_random:bool=False, id_random_length:int=8):
# Updated 2021-09-07
def sql_insert(
sql: str|None = None,
data: dict|None = None,
table_name: str|None = None,
rm_id_random: bool = False,
id_random_length: int = 8,
):
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -76,39 +83,39 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
# http://sqlalche.me/e/14/gkpj
# Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'"
trans.rollback()
log.error('This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****')
return None
except OperationalError as e: # Likely an unknown field or related
trans.rollback()
log.error('This is likely because there was an unknown field or similar included. Returning False')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False')
log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****')
return False
except Exception as e:
trans.rollback()
log.error('This is likely because there was an attempt to create a duplicate entry. Returning None')
log.error('An unknown exception happened. Returning False')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
@@ -121,24 +128,6 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
# print('**** *** ** * ### ### * ** *** ****')
# log.error(e.code)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.statement)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.params)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.orig)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.ismulti)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.hide_parameters)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.detail)
# log.error('**** *** ** * ### ### * ** *** ****')
# log.error(e.connection_invalidated)
# log.error('**** *** ** * ### ### * ** *** ****')
return False
else:
log.debug(result_insert)
@@ -167,7 +156,16 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
# ### BEGIN ### Core Help CRUD ### sql_update() ###
def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None, record_id:int|None=None, record_id_random:str|None=None, rm_id_random:bool=False, id_random_length:None|int=None):
# Updated 2021-09-07
def sql_update(
sql: str|None = None,
data: dict|None = None,
table_name: str|None = None,
record_id: int|None = None,
record_id_random: str|None = None,
rm_id_random: bool = False,
id_random_length: None|int = None
):
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -220,23 +218,39 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
# http://sqlalche.me/e/14/gkpj
# Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'"
trans.rollback()
log.error('This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****')
return None
except OperationalError as e: # Likely an unknown field or related
trans.rollback()
log.error('This is likely because there was an unknown field or similar included. Returning False')
log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False')
log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****')
return False
except Exception as e:
trans.rollback()
log.error('An unknown exception happened. Returning False')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
@@ -249,11 +263,6 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
return False
except Exception as e:
trans.rollback()
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
return False
else:
log.debug(result_update)
@@ -280,12 +289,13 @@ def sql_update(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
# ### BEGIN ### Core Help CRUD ### sql_insert_or_update() ###
# The catch all SQL INSERT or UPDATE function - STI 2021-02-17
# This one does it all for SQL INSERT and UPDATE queries
# Updated 2021-09-07
def sql_insert_or_update(
sql: str|None = None,
data: dict|None = None,
table_name: str|None = None,
rm_id_random: bool = False,
id_random_length: int|None = None
id_random_length: int|None = None,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -343,40 +353,49 @@ def sql_insert_or_update(
# http://sqlalche.me/e/14/gkpj
# Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'"
trans.rollback()
log.error('This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('An integrity error exception happened. This is likely because there was an attempt to create a duplicate entry. Returning None')
log.exception('**** *** ** * ### BEGIN ### Integrity Error Exception Happened: Returning None * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Integrity Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### BEGIN ### Integrity Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### END ### Integrity Error Exception Details: * ** *** ****')
return None
except OperationalError as e: # Likely an unknown field or related
trans.rollback()
log.error('This is likely because there was an unknown field or similar included. Returning False')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('An operational error exception happened. This is likely because there was an unknown field or similar included. Returning False')
log.exception('**** *** ** * ### BEGIN ### Operational Error Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Error Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### BEGIN ### Operational Error Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
log.error('**** *** ** * ### END ### Operational Error Exception Details: * ** *** ****')
return False
except Exception as e:
trans.rollback()
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
return False
else:
log.debug(result_insert_or_update)
@@ -406,6 +425,7 @@ def sql_insert_or_update(
# ### BEGIN ### Core Help CRUD ### sql_select() ###
# The catch all SQL SELECT function - STI 2021-02-17
# This one does it all for SQL SELECT queries
# Updated 2021-09-07
def sql_select(
table_name: str|None = None,
record_id: int|None = None,
@@ -417,7 +437,7 @@ def sql_select(
rm_id_random: bool = False,
as_dict: bool|None = True,
as_list: bool|None = False,
max_count: int = 100000
max_count: int = 100000,
):
current_log_level = log.level
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -536,14 +556,11 @@ def sql_select(
log.info('Executing with SQL statement only...')
result = db.execute(sql)
except OperationalError as e:
log.warning('*** An exception happened: OperationalError ***')
log.warning('* This is likely a "MySQL server has gone away" error. Going to try again... *')
log.warning(repr(e))
log.warning('***')
log.warning(str(e))
log.warning('^^^ exception ^^^')
log.error('An operational error exception happended. This is likely a "MySQL server has gone away" error. Going to try again...')
log.exception('**** *** ** * ### BEGIN ### Operational Exception Happened: Trying again... * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Exception ^ ^^ ^^^ ^^^^')
log.warning('Trying to recreate the pool...')
log.info('Trying to recreate the pool...')
log.debug('############## ############')
log.debug(dir(db))
log.debug(vars(db))
@@ -555,15 +572,16 @@ def sql_select(
log.debug(vars(db.engine.pool))
log.debug('############## ############')
db.engine.dispose()
log.warning('Now trying the query again...')
log.info('Now trying the query again...')
try:
if data:
log.warning('2x Executing with SQL statement and data...')
log.info('2x Executing with SQL statement and data...')
result = db.execute(sql, data)
else:
log.warning('2x Executing with SQL statement only...')
log.info('2x Executing with SQL statement only...')
result = db.execute(sql)
except Exception as e:
log.error('Tried again an exception was raised again. Not going to try again.')
log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
return False # Not successful
@@ -571,9 +589,19 @@ def sql_select(
log.info('Successfully executed the SQL on the second try.')
pass
except Exception as e:
log.info('An exception happened. Returning False.')
log.error('An unknown exception happened. Returning False.')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
return False # Not successful
else:
log.info('Successfully executed the SQL on the first try.')
@@ -637,14 +665,15 @@ def sql_select(
# ### BEGIN ### Core Help CRUD ### sql_delete() ###
# The catch all SQL DELETE function - STI 2021-02-17
# This one does it all for SQL DELETE queries
# Updated 2021-09-07
def sql_delete(
table_name:str|None=None,
record_id:int|None=None,
record_id_random:str|None=None,
field_name:str|None=None,
field_value=None,
sql:str|None=None,
data:dict|None=None
table_name: str|None = None,
record_id: int|None = None,
record_id_random: str|None = None,
field_name: str|None = None,
field_value = None,
sql: str|None = None,
data: dict|None = None
):
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -728,15 +757,11 @@ def sql_delete(
log.debug(dir(result))
log.debug(vars(result))
except OperationalError as e:
log.warning('This is likely a "MySQL server has gone away" error. Going to try again...')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: OperationalError MySQL server connection? * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
# log.warning(repr(e))
# log.warning('***')
# log.warning(str(e))
# log.warning('^^^ exception ^^^')
log.error('An operational error exception happended. This is likely a "MySQL server has gone away" error. Going to try again...')
log.exception('**** *** ** * ### BEGIN ### Operational Exception Happened: Trying again... * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Operational Exception ^ ^^ ^^^ ^^^^')
log.warning('Trying to recreate the pool...')
log.info('Trying to recreate the pool...')
log.debug('############## ############')
log.debug(dir(db))
log.debug(vars(db))
@@ -748,27 +773,38 @@ def sql_delete(
log.debug(vars(db.engine.pool))
log.debug('############## ############')
db.engine.dispose()
log.warning('Now trying the query again...')
log.info('Now trying the query again...')
try:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
if data:
log.warning('2x Executing with SQL statement and data...')
log.info('2x Executing with SQL statement and data...')
result = db.execute(sql, data)
else:
log.warning('2x Executing with SQL statement only...')
log.info('2x Executing with SQL statement only...')
result = db.execute(sql)
log.debug(result)
except Exception as e:
log.warning('Tried again an exception was raised again. Not going to try again.')
log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: * ** *** ****')
log.error('Tried again an exception was raised again. Not going to try again.')
log.exception('**** *** ** * ### BEGIN ### (2x) Second Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
return False # Not successful
else:
log.info('Successfully executed the SQL on the second try.')
pass
except Exception as e:
log.error('An unknown exception happened. Returning False.')
log.exception('**** *** ** * ### BEGIN ### Exception Happened: Returning False * ** *** ****')
log.error('^^^^ ^^^ ^^ ^ ### END ### Exception ^ ^^ ^^^ ^^^^')
log.error('**** *** ** * ### BEGIN ### Exception Details: * ** *** ****')
log.error('**** *** ** * SQL Statement: * ** *** ****')
log.error(e.statement)
log.error('**** *** ** * SQL Parameters: * ** *** ****')
log.error(e.params)
log.error('**** *** ** * SQL Origin Message: * ** *** ****')
log.error(e.orig)
log.error('**** *** ** * ### END ### Exception Details: * ** *** ****')
return False # Not successful
else:
log.info('Successfully executed the SQL on the first try.')
@@ -776,9 +812,7 @@ def sql_delete(
# NOTE: Need to deal with 0 rows affected when the WHERE clause was not satisfied and there was no error.
return True # Successful
# ### END ### Core Help CRUD ### sql_delete() ###
# ### BEGIN ### API DB SQL ### redis_lookup_id_random() ###
@@ -788,7 +822,10 @@ def sql_delete(
# If success then return the ID number
# If not success and there is a table_name then check the database table passed
# If found in database table then store in Redis and return the ID number
def redis_lookup_id_random(record_id_random:int|str, table_name:str):
def redis_lookup_id_random(
record_id_random: int|str,
table_name: str,
):
current_log_level = log.level
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -873,7 +910,10 @@ def redis_lookup_id_random(record_id_random:int|str, table_name:str):
# ### BEGIN ### API DB SQL ### get_id_random() ###
# Changed name from lookup_id_random() to get_id_random()
# Updated 2021-08-23
def get_id_random(record_id:int, table_name:str):
def get_id_random(
record_id: int,
table_name: str
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -912,7 +952,9 @@ def get_id_random(record_id:int, table_name:str):
# Look up and resolve id_random values to their id
# Remove the unneeded *_id_random key from the dict
# This really needs to be simplified... Use a list of dicts instead. Can store as JSON in the DB.
def lookup_id_random_pop(obj_data:dict):
def lookup_id_random_pop(
obj_data: dict
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -1138,7 +1180,6 @@ def lookup_id_random_pop(obj_data:dict):
# ### END ### API DB SQL ### lookup_id_random_pop() ###
# ### BEGIN ### API DB SQL Methods ### get_account_id_w_for_type_id() ###
# Updated 2021-08-24
def get_account_id_w_for_type_id(
@@ -1169,4 +1210,4 @@ def get_account_id_w_for_type_id(
if account_id := for_data_result.get('account_id', None): return account_id
else: return False
else: return None
# ### END ### API DB SQL Methods ### get_account_id_w_for_type_id() ###
# ### END ### API DB SQL Methods ### get_account_id_w_for_type_id() ###