Improved SQL results to dict and list of dicts. Should be noticably more efficient under load.

app/db_sql.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 import secrets
+from timeit import default_timer as timer
+
 from app.config import settings
 from .log import *
 #from .lib_general import lookup_id_random_pop
@@ -353,13 +355,6 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
         log.info('Successfully executed the SQL on the first try.')
         pass
 
-    #log.debug(result.fetchall()) # Uncommenting this breaks things?
-    # BEGIN NOTE: Check this out later! ###
-    #header = result.keys()
-    #for row in result:
-    #    yield dict(zip(header, row))
-    # END NOTE: Check this out later! ###
-
     #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(result.rowcount)
     log.debug(vars(result))
@@ -367,7 +362,9 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
     if result.rowcount == 1:
         log.info(f'Found one record. as_dict={as_dict}, as_list={as_list}')
         if as_dict:
-            record = sql_result_proxy_to_dict_simple(result_proxy=result.first())
+            # After testing, this method is the fastest way to convert to a dict - STI 2021-03-09
+            # my custom sql_result_proxy_to_dict_simple(result_proxy=result.first()) is slower
+            record = dict(result.first())
         else:
             record = result.first()
         if as_list:
@@ -375,6 +372,7 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
             record_li.append(record)
             #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
             log.debug(record_li)
+
             return record_li # Successful
         else:
             #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -384,7 +382,16 @@ def sql_select(table_name=None, record_id=None, record_id_random=None, field_nam
         log.info(f'Found {result.rowcount} records. as_dict={as_dict}, as_list={as_list}')
         #log.info('Found more than one record. Returning as a list of dicts.')
         if as_dict:
-            record_li = sql_result_proxy_to_dict_simple(result_proxy=result.fetchall())
+            # After testing, this method is the fastest way to convert to a list of dicts - STI 2021-03-09
+            # list(result) was tested and seems to be the slowest
+            # my custom sql_result_proxy_to_dict_simple(result_proxy=result.fetchall()) was tested and is only slightly faster than list(result)
+            #timer_1_start = timer()
+            record_li = [dict(record) for record in result.fetchall()]
+            #log.debug(record_li)
+            #log.debug(type(record_li))
+            #log.debug(type(record_li[0]))
+            #timer_1_end = timer()
+            #log.debug( round((timer_1_end - timer_1_start), 8) )
         else:
             record_li = result.fetchall()
         log.debug(record_li)
@@ -534,41 +541,3 @@ def sql_delete(table_name:str=None, record_id:int=None, record_id_random:str=Non
 
     # NOTE: Need to deal with 0 rows affected when the WHERE clause was not satisfied and there was no error.
     return True # Successful
-
-
-# NOTE WARNING: This is a near duplicate of what is under lib_rest (was lib_general). WARNING
-# Change SQL SELECT result RowProxy record to a dict (named key/value)
-# Change SQL SELECT list result RowProxy records to a list of dicts (named key/value)
-def sql_result_proxy_to_dict_simple(result_proxy=None):
-    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
-    log.debug(locals())
-
-    log.debug(type(result_proxy))
-
-    if isinstance(result_proxy, list):
-        log.info('Processing a SQL list...')
-
-        record_li = []
-        for row_proxy in result_proxy:
-            log.debug(row_proxy)
-
-            record = {}
-            for key, value in row_proxy.items():
-                record[key] = value
-            record_li.append(record)
-        return record_li
-
-    # Must import sqlalchemy to check the type correctly.
-    # Or convert it to a string and compare.
-    if str(type(result_proxy)) == '<class \'sqlalchemy.engine.result.RowProxy\'>':
-    #if isinstance(result_proxy, sqlalchemy.engine.result.RowProxy):
-        log.info('Processing a SQL record (sqlalchemy.engine.result.RowProxy)')
-
-        row_proxy = result_proxy
-
-        record = {}
-        for key, value in row_proxy.items():
-            record[key] = value
-        return record
-    return False
-

app/routers/api_crud.py

@@ -133,7 +133,7 @@ async def get_obj_li(obj_type_l1: str=None,
                      by_alias: Optional[bool] = True,
                      exclude_unset: Optional[bool] = True,
                      ):
-    log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(locals())
 
     debug_data = {}
@@ -208,7 +208,7 @@ async def get_obj(obj_type_l1: str=None,
                   by_alias: Optional[bool] = True,
                   exclude_unset: Optional[bool] = True,
                   ):
-    log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(locals())
 
     debug_data = {}