Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improved SQL results to dict and list of dicts. Should be noticably more efficient under load.

Browse Source
This commit is contained in:
Scott Idem
2021-03-09 11:24:04 -05:00
parent be2beb04f0
commit bc22a8c9d8
2 changed files with 18 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
app/db_sql.py
View File

@@ -1,5 +1,7 @@
from __future__ import annotations
import secrets
from timeit import default_timer as timer
from app.config import settings
from .log import *
#from .lib_general import lookup_id_random_pop
@@ -353,13 +355,6 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
log.info('Successfully executed the SQL on the first try.')
pass
#log.debug(result.fetchall()) # Uncommenting this breaks things?
# BEGIN NOTE: Check this out later! ###
#header = result.keys()
#for row in result:
# yield dict(zip(header, row))
# END NOTE: Check this out later! ###
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(result.rowcount)
log.debug(vars(result))
@@ -367,7 +362,9 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
if result.rowcount == 1:
log.info(f'Found one record. as_dict={as_dict}, as_list={as_list}')
if as_dict:
record = sql_result_proxy_to_dict_simple(result_proxy=result.first())
# After testing, this method is the fastest way to convert to a dict - STI 2021-03-09
# my custom sql_result_proxy_to_dict_simple(result_proxy=result.first()) is slower
record = dict(result.first())
else:
record = result.first()
if as_list:
@@ -375,6 +372,7 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
record_li.append(record)
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(record_li)
return record_li # Successful
else:
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -384,7 +382,16 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
log.info(f'Found {result.rowcount} records. as_dict={as_dict}, as_list={as_list}')
#log.info('Found more than one record. Returning as a list of dicts.')
if as_dict:
record_li = sql_result_proxy_to_dict_simple(result_proxy=result.fetchall())
# After testing, this method is the fastest way to convert to a list of dicts - STI 2021-03-09
# list(result) was tested and seems to be the slowest
# my custom sql_result_proxy_to_dict_simple(result_proxy=result.fetchall()) was tested and is only slightly faster than list(result)
#timer_1_start = timer()
record_li = [dict(record) for record in result.fetchall()]
#log.debug(record_li)
#log.debug(type(record_li))
#log.debug(type(record_li[0]))
#timer_1_end = timer()
#log.debug( round((timer_1_end - timer_1_start), 8) )
else:
record_li = result.fetchall()
log.debug(record_li)
@@ -534,41 +541,3 @@ def sql_delete(table_name:str=None, record_id:int=None, record_id_random:str=Non
# NOTE: Need to deal with 0 rows affected when the WHERE clause was not satisfied and there was no error.
return True # Successful
# NOTE WARNING: This is a near duplicate of what is under lib_rest (was lib_general). WARNING
# Change SQL SELECT result RowProxy record to a dict (named key/value)
# Change SQL SELECT list result RowProxy records to a list of dicts (named key/value)
def sql_result_proxy_to_dict_simple(result_proxy=None):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
log.debug(type(result_proxy))
if isinstance(result_proxy, list):
log.info('Processing a SQL list...')
record_li = []
for row_proxy in result_proxy:
log.debug(row_proxy)
record = {}
for key, value in row_proxy.items():
record[key] = value
record_li.append(record)
return record_li
# Must import sqlalchemy to check the type correctly.
# Or convert it to a string and compare.
if str(type(result_proxy)) == '<class \'sqlalchemy.engine.result.RowProxy\'>':
#if isinstance(result_proxy, sqlalchemy.engine.result.RowProxy):
log.info('Processing a SQL record (sqlalchemy.engine.result.RowProxy)')
row_proxy = result_proxy
record = {}
for key, value in row_proxy.items():
record[key] = value
return record
return False

4
app/routers/api_crud.py
View File

@@ -133,7 +133,7 @@ async def get_obj_li(obj_type_l1: str=None,
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
debug_data = {}
@@ -208,7 +208,7 @@ async def get_obj(obj_type_l1: str=None,
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
debug_data = {}