From cfa58f2c32de5d25761f80995e8381afdf83b220 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 15 Feb 2022 11:33:35 -0500 Subject: [PATCH] General code clean up for debugging. Also work on event badge related. --- app/db_sql.py | 43 ++++++------ app/lib_general.py | 27 ++++---- app/log.py | 12 ++-- app/routers/event_badge.py | 113 ++++++++++++++----------------- app/routers/membership_person.py | 6 +- app/routers/person.py | 19 +++--- 6 files changed, 98 insertions(+), 122 deletions(-) diff --git a/app/db_sql.py b/app/db_sql.py index a700a60..2c50c59 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -175,9 +175,10 @@ def sql_update( record_id: int|None = None, record_id_random: str|None = None, rm_id_random: bool = False, - id_random_length: None|int = None + id_random_length: None|int = None, + log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(log_lvl) log.debug(locals()) if sql: @@ -458,9 +459,9 @@ def sql_select( as_dict: bool|None = True, as_list: bool|None = False, max_count: int = 100000, + log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL ) -> None|bool|dict|list: - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) + log.setLevel(log_lvl) if table_name and not (record_id or record_id_random or field_name or field_value or sql or data): # Select all records from a table @@ -543,11 +544,11 @@ def sql_select( sql = text(sql) elif sql and data and not (table_name or record_id or record_id_random or field_name or field_value): # Select records based on the SQL statement given and with the matching data dict fields and values + log.info('Select records based on the SQL statement given and with the matching data dict fields and values') if rm_id_random: data = lookup_id_random_pop(obj_data=data) - log.info('Select records based on the SQL statement given and with the matching data dict fields and values') sql = text(sql) else: # Nothing matched the expected combination of parameters passed to this function @@ -628,36 +629,34 @@ def sql_select( log.debug('Successfully executed the SQL on the first try.') pass + #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(f'Row count: {result.rowcount}') #log.debug(vars(result)) #log.debug(dir(result)) + + # NOTE: as_dict defaults to True for this function + # NOTE: as_list defaults to False for this function + # NOTE: After testing, this method is the fastest way to convert to a dict - STI 2021-03-09 + # NOTE: My custom sql_result_proxy_to_dict_simple(result_proxy=result.first()) is slower than using dict(). + # NOTE: list(result) was tested seems to be the slowest. Slower than my custom function. + if result.rowcount == 1: log.info(f'Found one record. as_dict={as_dict}, as_list={as_list}') if as_dict: - # After testing, this method is the fastest way to convert to a dict - STI 2021-03-09 - # my custom sql_result_proxy_to_dict_simple(result_proxy=result.first()) is slower record = dict(result.first()) else: record = result.first() if as_list: record_li = [] record_li.append(record) - #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(record_li) - return record_li # Successful else: - #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(record) - return record # Successful elif result.rowcount > 1: log.info(f'Found {result.rowcount} records. as_dict={as_dict}, as_list={as_list}') - #log.info('Found more than one record. Returning as a list of dicts.') if as_dict: - # After testing, this method is the fastest way to convert to a list of dicts - STI 2021-03-09 - # list(result) was tested and seems to be the slowest - # my custom sql_result_proxy_to_dict_simple(result_proxy=result.fetchall()) was tested and is only slightly faster than list(result) #timer_1_start = timer() record_li = [dict(record) for record in result.fetchall()] #log.debug(record_li) @@ -672,16 +671,14 @@ def sql_select( return record_li # Successful else: if as_list: - # log.info('No records found. Returning None since the list is an empty list.') log.info('No records found. Returning an empty list.') log.debug(result) - - return [] # Successful + return [] # Successful even though no results else: log.info('No records found. Returning None.') log.debug(result) - return None # Successful + return None # Successful even though no results # ### END ### Core Help CRUD ### sql_select() ### @@ -1264,7 +1261,7 @@ def get_account_id_w_for_type_id( # Updated 2022-01-17 @logger_reset def sql_enable_part(table_name: str, enabled: str) -> bool|dict: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if not table_name: return False @@ -1282,10 +1279,8 @@ def sql_enable_part(table_name: str, enabled: str) -> bool|dict: elif enabled == 'all': sql = f'AND (`{table_name}`.enable = true OR `{table_name}`.enable = false OR `{table_name}`.enable IS NULL)' enable = None - log.debug(sql) - log.debug(enable) - # return result + return sql, enable else: return False @@ -1296,7 +1291,7 @@ def sql_enable_part(table_name: str, enabled: str) -> bool|dict: # Updated 2022-01-17 @logger_reset def sql_limit_offset_part(limit: int, offset: int = 0) -> bool|str: - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if limit >= 0 and offset >= 0: diff --git a/app/lib_general.py b/app/lib_general.py index 4e220d1..a33d36a 100644 --- a/app/lib_general.py +++ b/app/lib_general.py @@ -1,4 +1,4 @@ -from __future__ import annotations +# from __future__ import annotations import datetime, html2text, jwt, os, pandas, pathlib, pytz, redis, time from passlib.hash import argon2 @@ -20,7 +20,7 @@ from app.db_sql import redis_lookup_id_random, sql_select # ### BEGIN ### API Lib General ### async get_token_header() ### -async def get_token_header(x_token: str = Header(...)): +def get_token_header(x_token: str = Header(...)): if x_token != 'fake-super-secret-token': raise HTTPException(status_code=400, detail='X-Token header invalid') # ### END ### API Lib General ### async get_token_header() ### @@ -52,9 +52,10 @@ class Common_Route_Params: # ### END ### API Lib General ### class Common_Route_Params ### -# ### BEGIN ### API Lib General ### async route_commons() ### -# Updated 2022-01-05 -async def common_route_params( +# ### BEGIN ### API Lib General ### common_route_params() ### +# Updated 2022-02-15 +@logger_reset # This breaks things for some reason when the function is async. Do not use async def common_route_params()! +def common_route_params( x_account_id: str = Header(..., min_length=11, max_length=22), enabled: str = 'enabled', # all, enabled, disabled limit: int = 100, @@ -66,23 +67,19 @@ async def common_route_params( # exclude: Optional[list] = [], # Leaving this and include commented out # include: Optional[list] = [], # Leaving this and exclude commented out response: Response = Response, + log_lvl: int = logging.WARNING, # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL ) -> Common_Route_Params: - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(log_lvl) log.debug(locals()) - log.info(f'Setting commons values') - - log.info(f'The x-account-id header has a value. x-account-id: {x_account_id}') + log.info(f'Setting commons values: x_account_id, x_account_id_random, limit, offset, enabled, by_alias, exclude_unset, response') x_account_id_random = x_account_id if x_account_id := redis_lookup_id_random(table_name='account', record_id_random=x_account_id): - log.info(f'Found the x-account-id with the value: {x_account_id}') - # x_account = { 'id': account_id, 'id_random': x_account_id } - # log.debug(x_account) - # return x_account + log.info(f'Found the x-account-id header with the value: {x_account_id}') else: - log.warning(f'The x-account-id Account ID was not found. Account ID: {x_account_id}') + log.warning(f'The x-account-id header was found, but the Account ID was not found or is not valid. Account ID: {x_account_id}') raise HTTPException(status_code=403, detail='The x-account-id Account ID was not found.') # Forbidden commons = Common_Route_Params( x_account_id=x_account_id, x_account_id_random=x_account_id_random, limit=limit, offset=offset, enabled=enabled, by_alias=by_alias, exclude_unset=exclude_unset, response=response ) @@ -90,7 +87,7 @@ async def common_route_params( log.debug(commons) return commons -# ### END ### API Lib General ### async route_commons() ### +# ### END ### API Lib General ### async common_route_params() ### def secure_hash_string(string: str): diff --git a/app/log.py b/app/log.py index f3e29ba..08229ab 100644 --- a/app/log.py +++ b/app/log.py @@ -10,20 +10,18 @@ logging.basicConfig( # ### BEGIN ### Log ### logger_reset() ### # https://realpython.com/primer-on-python-decorators/ -# Updated 2021-09-29 +# Updated 2022-02-15 def logger_reset(func): # log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - # log.debug(locals()) + # log.info(locals()) @functools.wraps(func) def wrapper(*args, **kwargs): - # log.info(f'Function "{func.__name__}()" called. Saving logger level... Log level: {log.level}') + log.info(f'*** Function: "{func.__name__}()"') + log.debug(f'*** Function Positional Args: {args}\nFunction Key Args: {kwargs}') init_log_level = log.level returned_result = func(*args, **kwargs) - log.info(f'*** Function "{func.__name__}()" finished. Resetting logger level to level: {log.level} ***') + log.debug(f'*** Function finished: "{func.__name__}()". Resetting logger level to level: {log.level} ***') log.setLevel(init_log_level) - # log.info(f'Reset logger level. Log level: {log.level}') - # return func(*args, **kwargs) return returned_result - return wrapper # ### END ### Log ### logger_reset() ### \ No newline at end of file diff --git a/app/routers/event_badge.py b/app/routers/event_badge.py index f9e87a5..86db293 100644 --- a/app/routers/event_badge.py +++ b/app/routers/event_badge.py @@ -3,9 +3,9 @@ from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Resp from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union -from app.lib_general import log, logging +from app.lib_general import log, logging, common_route_params, Common_Route_Params from app.config import settings -from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random +from app.db_sql import sql_enable_part, sql_insert, sql_update, sql_insert_or_update, sql_limit_offset_part, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template @@ -135,70 +135,51 @@ async def get_event_badge_obj_li( # ### BEGIN ### API Event Badge ### search_event_badge_obj_li() ### -# Updated 2021-07-28 +# Updated 2022-02-15 @router.get('/event/{event_id}/badge/search', response_model=Resp_Body_Base) async def search_event_badge_obj_li( - # account_id: str = Query(None, min_length=11, max_length=22), event_id: str = Query(None, min_length=11, max_length=22), event_badge_id: str = Query(None, min_length=3, max_length=22), event_person_id: str = Query(None, min_length=3, max_length=22), - external_id: str = Query('%', max_length=50), + + external_id: str = Query('%', max_length=100), # NOTE: Not currently used! 2022-02-15 + given_name: str = Query('%', max_length=25), family_name: str = Query('%', max_length=25), email: str = Query('%', max_length=50), - limit: int = 50, - enabled: str = 'enabled', - x_account_id: str = Header(...), - by_alias: bool = True, - exclude_unset: bool = True, - response: Response = Response, + + commons: Common_Route_Params = Depends(common_route_params), ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - if account_id := redis_lookup_id_random(record_id_random=x_account_id, table_name='account'): pass - else: return mk_resp(data=None, status_code=404, response=response) + # account_id = commons.x_account_id if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass - else: return mk_resp(data=None, status_code=404, response=response) + else: return mk_resp(data=None, status_code=404, response=commons.response) + + log.info(f'Searching: Event ID: {event_id}; Event Badge ID: {event_badge_id}; Event Person ID: {event_person_id}; External ID: {external_id}; Give Name: {given_name}; Family Name: {family_name}; Email: {email}') data = {} - data['account_id'] = account_id + # data['account_id'] = account_id data['event_id'] = event_id - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(event_badge_id) - if event_badge_id and len(event_badge_id) > 2: - log.info(f'Found event_badge_id: {event_badge_id}') + log.info(f'Using Event Badge ID: {event_badge_id}') data['event_badge_id_random'] = f'%{event_badge_id}%' elif event_person_id and len(event_person_id) > 2: - log.info(f'Found event_person_id: {event_person_id}') + log.info(f'Using Event Person ID: {event_person_id}') data['event_person_id_random'] = f'%{event_person_id}%' else: + log.info(f'Using other data fields') data['external_id'] = '%'+external_id+'%' data['given_name'] = '%'+given_name+'%' data['family_name'] = '%'+family_name+'%' data['email'] = '%'+email+'%' # Adding the % symbol here because it turns certain combinations into special characters - # data['from_datetime'] = from_datetime - # data['to_datetime'] = to_datetime log.debug(data) - if enabled in ['enabled', 'disabled', 'all']: - if enabled == 'enabled': - data['enable'] = True - sql_enabled = f'AND `event_badge`.enable = :enable' - elif enabled == 'disabled': - data['enable'] = False - sql_enabled = f'AND `event_badge`.enable = :enable' - elif enabled == 'all': - sql_enabled = '' - - if limit: - data['limit'] = limit - sql_limit = f'LIMIT :limit' - else: - sql_limit = '' + sql_enabled, data['enable'] = sql_enable_part(table_name='event_badge', enabled=commons.enabled) # Reasonably safe return str and bool + sql_limit = sql_limit_offset_part(limit=commons.limit, offset=commons.offset) # Reasonably safe return str if event_badge_id and len(event_badge_id) > 2: sql = f""" @@ -244,51 +225,55 @@ async def search_event_badge_obj_li( ORDER BY event_badge.given_name ASC, event_badge.family_name ASC, event_badge.created_on DESC, event_badge.updated_on DESC {sql_limit}; """ - log.debug(sql) - if sql_result := sql_select(data=data, sql=sql, as_list=True, rm_id_random=False): # rm_id_random=True + + if sql_result := sql_select(data=data, sql=sql, as_list=True, rm_id_random=False, log_lvl=logging.INFO): # rm_id_random=True resp_data_li = [] for record in sql_result: - resp_data = Event_Badge_Base(**record).dict(by_alias=by_alias, exclude_unset=exclude_unset) + resp_data = Event_Badge_Base(**record).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) resp_data_li.append(resp_data) - - return mk_resp(data=resp_data_li, response=response) + return mk_resp(data=resp_data_li, response=commons.response) + elif isinstance(sql_result, list) or sql_result is None: # Empty list or None + log.info('No results') + return mk_resp(data=None, status_code=404, response=commons.response) # Not Found else: - log.debug(sql_result) - return mk_resp(data=None, status_code=404, response=response) + log.warning('Likely bad request') + return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request # ### END ### API Event Badge ### search_event_badge_obj_li() ### # ### BEGIN ### API Event Badge ### get_event_badge_obj() ### -# Updated 2021-08-05 +# Updated 2022-02-15 @router.get('/event/badge/{event_badge_id}', response_model=Resp_Body_Base) async def get_event_badge_obj( event_badge_id: str = Query(..., min_length=11, max_length=22), - limit: int = 5, # For now this covers any included objects or object lists - enabled: str = 'enabled', # For now this covers any included objects or object lists inc_event_badge_template: bool = True, - x_account_id: str = Header(...), - by_alias: Optional[bool] = True, - exclude_unset: Optional[bool] = True, - response: Response = Response, + + commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if event_badge_id := redis_lookup_id_random(record_id_random=event_badge_id, table_name='event_badge'): pass - else: return mk_resp(data=None, status_code=404, response=response) + else: return mk_resp(data=None, status_code=404, response=commons.response) - event_badge_obj = load_event_badge_obj( - event_badge_id = event_badge_id, - limit = limit, - by_alias = by_alias, - exclude_unset = exclude_unset, - # model_as_dict = model_as_dict, - enabled = enabled, - inc_event_badge_template = inc_event_badge_template, - ) - event_badge_dict = event_badge_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) - return mk_resp(data=event_badge_dict, response=response) + if event_badge_obj := load_event_badge_obj( + event_badge_id = event_badge_id, + inc_event_badge_template = inc_event_badge_template, + + # limit = commons.limit, + # by_alias = commons.by_alias, + # exclude_unset = commons.exclude_unset, + # model_as_dict = model_as_dict, + # enabled = commons.enabled, + ): + # event_badge_dict = event_badge_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) + # return mk_resp(data=event_badge_dict, response=commons.response) + response_data = event_badge_obj + else: + return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request + + return mk_resp(data=response_data, response=commons.response) # ### END ### API Event Badge ### get_event_badge_obj() ### diff --git a/app/routers/membership_person.py b/app/routers/membership_person.py index 4d6ea98..2cb8ac6 100644 --- a/app/routers/membership_person.py +++ b/app/routers/membership_person.py @@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, common_route_params, Common_Route_Params from app.config import settings -from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random +from app.db_sql import sql_enable_part, sql_insert, sql_update, sql_insert_or_update, sql_limit_offset_part, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template @@ -401,7 +401,7 @@ async def lookup_membership_person_obj( inc_membership_person_group_list = inc_membership_person_group_list, inc_membership_person_profile = inc_membership_person_profile, inc_membership_person_type = inc_membership_person_type, - ).dict(by_alias=by_alias, exclude_unset=exclude_unset) + ).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) data = membership_person_obj elif isinstance(membership_person_obj_result, list): membership_person_obj_li = [] @@ -414,7 +414,7 @@ async def lookup_membership_person_obj( inc_membership_person_group_list = inc_membership_person_group_list, inc_membership_person_profile = inc_membership_person_profile, inc_membership_person_type = inc_membership_person_type, - ).dict(by_alias=by_alias, exclude_unset=exclude_unset) + ).dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) ) data = membership_person_obj_li else: diff --git a/app/routers/person.py b/app/routers/person.py index d13dac5..bed3477 100644 --- a/app/routers/person.py +++ b/app/routers/person.py @@ -195,15 +195,15 @@ async def person_obj_external_id( person_id = person_data.get('person_id') if person_dict := load_person_obj( - person_id = person_id, - limit = commons.limit, - model_as_dict = True, # NOTE: returning model as a dict - enabled = commons.enabled, - inc_address = inc_address, - inc_contact = inc_contact, - inc_user = inc_user, - inc_user_role_list = inc_user_role_list, - ): + person_id = person_id, + limit = commons.limit, + model_as_dict = True, # NOTE: returning model as a dict + enabled = commons.enabled, + inc_address = inc_address, + inc_contact = inc_contact, + inc_user = inc_user, + inc_user_role_list = inc_user_role_list, + ): if isinstance(person_dict, dict): response_data = person_dict else: @@ -338,6 +338,7 @@ async def get_person_obj( inc_post_list: bool = False, # Priority l2 inc_post_comment_list: bool = False, # Priority l3 inc_user: bool = False, # Priority l1 + commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL