From c625cda8451581e622cc05e889a3610bf9a7032b Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Wed, 23 Mar 2022 15:38:53 -0400 Subject: [PATCH] More improvement to Impexium API calls. Should be faster and more robust now. --- app/methods/e_impexium_methods.py | 101 ++++++++++++++++------------- app/methods/event_badge_methods.py | 3 +- app/models/event_badge_models.py | 3 +- app/models/event_person_models.py | 3 +- app/routers/e_impexium.py | 65 ++++++++++++++++--- 5 files changed, 120 insertions(+), 55 deletions(-) diff --git a/app/methods/e_impexium_methods.py b/app/methods/e_impexium_methods.py index 6941390..cc13f50 100644 --- a/app/methods/e_impexium_methods.py +++ b/app/methods/e_impexium_methods.py @@ -26,10 +26,10 @@ api['access_token'] = None # ### BEGIN ### API External Impexium Methods ### get_access_token() ### -# Updated 2022-02-18 +# Updated 2022-03-23 @logger_reset def get_access_token(): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) log.debug(f'App data:\n{app}') @@ -49,6 +49,7 @@ def get_access_token(): log.debug(f'JSON: {resp.json()}') else: log.warning('No JSON data found') + return False # log.debug('Text:', resp.text) response_data = resp.json() @@ -60,21 +61,24 @@ def get_access_token(): log.debug(api) - return api + return True # ### END ### API External Impexium Methods ### get_access_token() ### # ### BEGIN ### API External Impexium Methods ### authenticate() ### -# Updated 2022-03-22 +# Updated 2022-03-23 @logger_reset def authenticate(): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if api.get('access_token'): pass else: - result = get_access_token() - log.debug(result) + log.warning('Access token not found. Calling get_access_token()...') + if result := get_access_token(): + log.debug(result) + else: + return False log.debug(f'App data:\n{app}') log.debug(f'API data:\n{api}') @@ -122,7 +126,6 @@ def authenticate(): # ### END ### API External Impexium Methods ### authenticate() ### - # ### BEGIN ### API External Impexium Methods ### get_custom_fields() ### # Updated 2022-02-18 @logger_reset @@ -189,6 +192,7 @@ def get_events(page=1): # ### BEGIN ### API External Impexium Methods ### get_event_registrants() ### # Updated 2021-10-07 +@logger_reset def get_event_registrants( event_code: str, registered_since: datetime.datetime = None, @@ -196,11 +200,13 @@ def get_event_registrants( page: int = 0, # return_all: bool = False ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - result = authenticate() - log.debug(result) + if result := authenticate(): + log.debug(result) + else: + return False try_page = True @@ -214,7 +220,7 @@ def get_event_registrants( impexium_event_registration_list = [] while try_page and page_num <= max_page: page_num = page_num + 1 - log.info(f'Getting page number {page_num}') + log.info(f'Getting page number {page_num} from Impexium... Event Code: {event_code}; Details: {details}; Registered Since: {registered_since}') endpoint = f'/Events/{event_code}/Registrations/{page_num}' uri = api['base_url']+endpoint @@ -241,16 +247,13 @@ def get_event_registrants( response_data = resp.json() current_page = response_data.get('pageNumber') - log.info(f'Current Page: {current_page}') + log.info(f'Impexium response Current Page: {current_page}') impexium_event_registration_list = impexium_event_registration_list + response_data.get('dataList') - # page = current_page + 1 - - # log.warning('Avoiding rate limit. Sleeping for .1 seconds...') - # time.sleep(.1) try_request = False elif resp.status_code == 404: + log.info('No results returned.') try_request = False try_page = False elif resp.status_code == 429: @@ -263,24 +266,13 @@ def get_event_registrants( try_page = False impexium_event_registration_list = False - # if current_page >= 1: - # impexium_event_registration_list = impexium_event_registration_list + response_data.get('dataList') - - # page = current_page + 1 - - # log.warning('Avoiding rate limit. Sleeping for .1 seconds...') - # time.sleep(.1) - # else: - # try_request = False - - # loop_count = loop_count + 1 - return impexium_event_registration_list # ### END ### API Impexium Methods ### get_event_registrants() ### # ### BEGIN ### API External Impexium Methods ### get_individual_registrations() ### # Updated 2022-02-18 +@logger_reset def get_individual_registrations(api, record_number, event_code=None, page=1): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -312,6 +304,7 @@ def get_individual_registrations(api, record_number, event_code=None, page=1): # ### BEGIN ### API External Impexium Methods ### get_individual_purchases() ### # Updated 2022-02-18 +@logger_reset def get_individual_purchases(api, record_number, product_code=None, purchased_since=None, product_category_code=None, page=1): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -342,38 +335,58 @@ def get_individual_purchases(api, record_number, product_code=None, purchased_si # ### BEGIN ### API External Impexium Methods ### get_individual_custom_fields() ### -# Updated 2022-02-18 -def get_individual_custom_fields(api, individual_id): +# Updated 2022-03-23 +@logger_reset +def get_individual_custom_fields( + individual_id + ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) endpoint = f'/Individuals/{individual_id}/CustomFields' uri = api['base_url']+endpoint - params = { } - resp = requests.get(url=uri, params=params, headers=api['headers']) + impexium_individual_custom_field_list = None - log.debug(f'Status Code: {resp.status_code}') - log.debug(f'Headers: {resp.headers}') - # log.debug(f'Encoding: {resp.encoding}') - log.debug(f'JSON: {resp.json()}') - # log.debug('Text:', resp.text) + try_request = True + max_tries = 5 + try_count = 0 + while try_request and try_count <= max_tries: + try_count = try_count + 1 - response_data = resp.json() + resp = requests.get(url=uri, params=params, headers=api['headers']) - # pp = pprint.PrettyPrinter(indent=2) - # pp.pprint(response_data) - # print('**************************') + log.debug(f'Status Code: {resp.status_code}') + log.debug(f'Headers: {resp.headers}') + # log.debug(f'Encoding: {resp.encoding}') + # log.debug('Text:') + # log.debug(resp.text) - individual_custom_field_li = response_data + if resp.status_code == 200: + log.debug(resp.json) + impexium_individual_custom_field_list = resp.json() + try_request = False + elif resp.status_code == 404: + log.info('No results returned.') + try_request = False + impexium_individual_custom_field_list = [] + elif resp.status_code == 429: + log.warning('Hit rate limit. Sleeping for .1 seconds...') + time.sleep(.1) + try_request = True + impexium_individual_custom_field_list = False + else: + try_request = False + impexium_event_registration_list = False - return individual_custom_field_li + return impexium_individual_custom_field_list # ### END ### API External Impexium Methods ### get_individual_custom_fields() ### # ### BEGIN ### API External Impexium Methods ### get_individual_orders_open() ### # Updated 2022-02-18 +@logger_reset def get_individual_orders_open(api, record_number, include_line_items=False, from_date=None, to_date=None, page=1): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) diff --git a/app/methods/event_badge_methods.py b/app/methods/event_badge_methods.py index a13bff5..c08f61b 100644 --- a/app/methods/event_badge_methods.py +++ b/app/methods/event_badge_methods.py @@ -98,6 +98,7 @@ def get_event_badge_template_id_w_event_id( # ### BEGIN ### API Event Badge Methods ### create_update_event_badge_obj_v4() ### # Updated 2022-02-23 +@logger_reset def create_update_event_badge_obj_v4( event_badge_dict_obj: Event_Badge_Base|dict, event_badge_id: int|str = None, @@ -108,7 +109,7 @@ def create_update_event_badge_obj_v4( fail_any: bool = False, # Fail if any thing goes wrong for sub objects return_outline: bool = False, ) -> int|bool: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # ### SECTION ### Secondary data validation diff --git a/app/models/event_badge_models.py b/app/models/event_badge_models.py index 77dbba9..92bc5da 100644 --- a/app/models/event_badge_models.py +++ b/app/models/event_badge_models.py @@ -43,7 +43,8 @@ class Event_Badge_Base(BaseModel): person_id_random: Optional[str] person_id: Optional[int] - external_id: Optional[str] + external_id: Optional[str] # Generated internally or externally. Needs to be stable. It should not change. + external_sys_id: Optional[str] # Generated by external system (should be stable and not change) pronouns: Optional[str] # Preferred pronouns informal_name: Optional[str] diff --git a/app/models/event_person_models.py b/app/models/event_person_models.py index 095c21b..584c129 100644 --- a/app/models/event_person_models.py +++ b/app/models/event_person_models.py @@ -56,7 +56,8 @@ class Event_Person_Base(BaseModel): user_id_random: Optional[str] user_id: Optional[int] - external_id: Optional[str] + external_id: Optional[str] # Generated internally or externally. Needs to be stable. It should not change. + external_sys_id: Optional[str] # Generated by external system (should be stable and not change) file_count: Optional[int] diff --git a/app/routers/e_impexium.py b/app/routers/e_impexium.py index 4b7c733..a9308fd 100644 --- a/app/routers/e_impexium.py +++ b/app/routers/e_impexium.py @@ -10,7 +10,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template -from app.methods.e_impexium_methods import get_custom_fields, get_events, get_event_registrants +from app.methods.e_impexium_methods import get_custom_fields, get_events, get_event_registrants, get_individual_custom_fields from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3 @@ -28,6 +28,7 @@ async def event_import_reg( e_impexium_event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: 42_AM (2022-04); EX22_AM (2022-04); 41V_2 (2021-04) registered_since: datetime.datetime = None, # datetime.datetime.now() + datetime.timedelta(seconds=120) details: bool = True, + inc_membership_fields: bool = True, page: int = 0, # 250 per page from Impexium event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: ZDzTBlevhZs (2022-04) # Account ID For ISHLT: d8TqXqf1EOg @@ -54,17 +55,19 @@ async def event_import_reg( for event_registrant in event_registrant_li: if loop_count > loop_limit: break log.info(f"Record Num: {event_registrant.get('recordNumber')}, Registration Num: {event_registrant.get('registrationNumber')}, Registrant Type Code: {event_registrant.get('registrantTypeCode')}") - log.debug(event_registrant) + # log.debug(event_registrant) # Creating an external ID from the Impexium record number and registration number. Very explicitly converting each number to a string before joining, just in case. external_id_old = str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber')) external_id_new = str(e_impexium_event_id)+':'+str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber')) + external_sys_id = event_registrant.get('id') # Impexium UUID for person event_person_data = {} event_person_data['account_id'] = account_id event_person_data['event_id'] = event_id event_person_data['external_id'] = external_id_new - event_person_data['external_id_old'] = external_id_old + event_person_data['external_sys_id'] = external_sys_id + # event_person_data['external_id_old'] = external_id_old email = None country_subdivision_code = None @@ -88,6 +91,20 @@ async def event_import_reg( country_alpha_2_code = country_data.get('twoLetterIsoCode') country = address.get('country') + degrees = None + organization_name = None + if inc_membership_fields: + individual_custom_fields_li = get_individual_custom_fields(individual_id=event_registrant.get('id')) + + for individual_custom_field in individual_custom_fields_li: + if individual_custom_field.get('name') == 'degree': + degrees = individual_custom_field.get('value') + if individual_custom_field.get('name') == 'organization_name': + organization_name = individual_custom_field.get('value') + + log.debug(f'Degrees: {degrees}; Organization Name: {organization_name}') + + event_person_profile_data = {} event_person_profile_data['pronouns'] = event_registrant.get('gender') event_person_profile_data['informal_name'] = event_registrant.get('preferredFirstName') @@ -98,11 +115,17 @@ async def event_import_reg( event_person_profile_data['family_name'] = event_registrant.get('lastName') event_person_profile_data['designations'] = event_registrant.get('suffix') - event_person_profile_data['professional_title'] = event_registrant.get('title') + if degrees: + event_person_profile_data['professional_title'] = degrees # Ideally should be degrees for ISHLT + else: + event_person_profile_data['professional_title'] = event_registrant.get('title') # Should this be None if no degrees found above??? event_person_profile_data['display_name'] = event_registrant.get('badgeName') - event_person_profile_data['affiliations'] = event_registrant.get('badgeOrganization') + if organization_name: + event_person_profile_data['affiliations'] = organization_name # Ideally should be organization_name for ISHLT + else: + event_person_profile_data['affiliations'] = event_registrant.get('badgeOrganization') # Should this be None if no organization_name found above??? if email: event_person_profile_data['email'] = email @@ -112,7 +135,8 @@ async def event_import_reg( event_badge_data = {} event_badge_data['event_id'] = event_id event_badge_data['external_id'] = external_id_new - event_badge_data['external_id_old'] = external_id_old + event_badge_data['external_sys_id'] = external_sys_id + # event_badge_data['external_id_old'] = external_id_old if reg_type_code := event_registrant.get('registrantTypeCode'): event_badge_data['badge_type_code'] = event_registrant.get('registrantTypeCode') # Using this as the badge_type @@ -204,11 +228,17 @@ async def event_import_reg( event_badge_data['family_name'] = event_registrant.get('lastName') event_badge_data['designations'] = event_registrant.get('suffix') - event_badge_data['professional_title'] = event_registrant.get('title') + if degrees: + event_badge_data['professional_title'] = degrees # Ideally should be degrees for ISHLT + else: + event_badge_data['professional_title'] = event_registrant.get('title') # Should this be None if no degrees found above??? event_badge_data['display_name'] = event_registrant.get('badgeName') - event_badge_data['affiliations'] = event_registrant.get('badgeOrganization') + if organization_name: + event_badge_data['affiliations'] = organization_name # Ideally should be organization_name for ISHLT + else: + event_badge_data['affiliations'] = event_registrant.get('badgeOrganization') # Should this be None if no organization_name found above??? if email: event_badge_data['email'] = email @@ -370,6 +400,25 @@ async def testing( ) log.debug(type(event_registrant_li)) + + for event_registrant in event_registrant_li: + log.debug(event_registrant) + log.info(f"ID: {event_registrant.get('id')}, Registration Num: {event_registrant.get('registrationNumber')}, Registrant Name: {event_registrant.get('badgeName')}") + + individual_custom_fields_li = get_individual_custom_fields(individual_id=event_registrant.get('id')) + # log.debug(individual_custom_fields_li) + degrees = None + organization_name = None + for individual_custom_field in individual_custom_fields_li: + if individual_custom_field.get('name') == 'degree': + degrees = individual_custom_field.get('value') + if individual_custom_field.get('organization_name') == 'organization_name': + organization_name = individual_custom_field.get('value') + + log.debug(f'Degree: {degrees}; Organization Name: {organization_name}') + + + if event_registrant_li: return mk_resp(data=event_registrant_li, status_code=200, response=commons.response) # return mk_resp(data=len(event_registrant_li), status_code=200, response=commons.response)