Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More improvement to Impexium API calls. Should be faster and more robust now.

Browse Source
This commit is contained in:
Scott Idem
2022-03-23 15:38:53 -04:00
parent 7c7bd3c530
commit c625cda845
5 changed files with 120 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
app/methods/e_impexium_methods.py
View File

@@ -26,10 +26,10 @@ api['access_token'] = None
# ### BEGIN ### API External Impexium Methods ### get_access_token() ### # ### BEGIN ### API External Impexium Methods ### get_access_token() ###
# Updated 2022-02-18 # Updated 2022-03-23
@logger_reset @logger_reset
def get_access_token(): def get_access_token():
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
log.debug(f'App data:\n{app}') log.debug(f'App data:\n{app}')
@@ -49,6 +49,7 @@ def get_access_token():
log.debug(f'JSON: {resp.json()}') log.debug(f'JSON: {resp.json()}')
else: else:
log.warning('No JSON data found') log.warning('No JSON data found')
return False
# log.debug('Text:', resp.text) # log.debug('Text:', resp.text)
response_data = resp.json() response_data = resp.json()
@@ -60,21 +61,24 @@ def get_access_token():
log.debug(api) log.debug(api)
return api return True
# ### END ### API External Impexium Methods ### get_access_token() ### # ### END ### API External Impexium Methods ### get_access_token() ###
# ### BEGIN ### API External Impexium Methods ### authenticate() ### # ### BEGIN ### API External Impexium Methods ### authenticate() ###
# Updated 2022-03-22 # Updated 2022-03-23
@logger_reset @logger_reset
def authenticate(): def authenticate():
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
if api.get('access_token'): pass if api.get('access_token'): pass
else: else:
result = get_access_token() log.warning('Access token not found. Calling get_access_token()...')
log.debug(result) if result := get_access_token():
log.debug(result)
else:
return False
log.debug(f'App data:\n{app}') log.debug(f'App data:\n{app}')
log.debug(f'API data:\n{api}') log.debug(f'API data:\n{api}')
@@ -122,7 +126,6 @@ def authenticate():
# ### END ### API External Impexium Methods ### authenticate() ### # ### END ### API External Impexium Methods ### authenticate() ###
# ### BEGIN ### API External Impexium Methods ### get_custom_fields() ### # ### BEGIN ### API External Impexium Methods ### get_custom_fields() ###
# Updated 2022-02-18 # Updated 2022-02-18
@logger_reset @logger_reset
@@ -189,6 +192,7 @@ def get_events(page=1):
# ### BEGIN ### API External Impexium Methods ### get_event_registrants() ### # ### BEGIN ### API External Impexium Methods ### get_event_registrants() ###
# Updated 2021-10-07 # Updated 2021-10-07
@logger_reset
def get_event_registrants( def get_event_registrants(
event_code: str, event_code: str,
registered_since: datetime.datetime = None, registered_since: datetime.datetime = None,
@@ -196,11 +200,13 @@ def get_event_registrants(
page: int = 0, page: int = 0,
# return_all: bool = False # return_all: bool = False
): ):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
result = authenticate() if result := authenticate():
log.debug(result) log.debug(result)
else:
return False
try_page = True try_page = True
@@ -214,7 +220,7 @@ def get_event_registrants(
impexium_event_registration_list = [] impexium_event_registration_list = []
while try_page and page_num <= max_page: while try_page and page_num <= max_page:
page_num = page_num + 1 page_num = page_num + 1
log.info(f'Getting page number {page_num}') log.info(f'Getting page number {page_num} from Impexium... Event Code: {event_code}; Details: {details}; Registered Since: {registered_since}')
endpoint = f'/Events/{event_code}/Registrations/{page_num}' endpoint = f'/Events/{event_code}/Registrations/{page_num}'
uri = api['base_url']+endpoint uri = api['base_url']+endpoint
@@ -241,16 +247,13 @@ def get_event_registrants(
response_data = resp.json() response_data = resp.json()
current_page = response_data.get('pageNumber') current_page = response_data.get('pageNumber')
log.info(f'Current Page: {current_page}') log.info(f'Impexium response Current Page: {current_page}')
impexium_event_registration_list = impexium_event_registration_list + response_data.get('dataList') impexium_event_registration_list = impexium_event_registration_list + response_data.get('dataList')
# page = current_page + 1
# log.warning('Avoiding rate limit. Sleeping for .1 seconds...')
# time.sleep(.1)
try_request = False try_request = False
elif resp.status_code == 404: elif resp.status_code == 404:
log.info('No results returned.')
try_request = False try_request = False
try_page = False try_page = False
elif resp.status_code == 429: elif resp.status_code == 429:
@@ -263,24 +266,13 @@ def get_event_registrants(
try_page = False try_page = False
impexium_event_registration_list = False impexium_event_registration_list = False
# if current_page >= 1:
# impexium_event_registration_list = impexium_event_registration_list + response_data.get('dataList')
# page = current_page + 1
# log.warning('Avoiding rate limit. Sleeping for .1 seconds...')
# time.sleep(.1)
# else:
# try_request = False
# loop_count = loop_count + 1
return impexium_event_registration_list return impexium_event_registration_list
# ### END ### API Impexium Methods ### get_event_registrants() ### # ### END ### API Impexium Methods ### get_event_registrants() ###
# ### BEGIN ### API External Impexium Methods ### get_individual_registrations() ### # ### BEGIN ### API External Impexium Methods ### get_individual_registrations() ###
# Updated 2022-02-18 # Updated 2022-02-18
@logger_reset
def get_individual_registrations(api, record_number, event_code=None, page=1): def get_individual_registrations(api, record_number, event_code=None, page=1):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
@@ -312,6 +304,7 @@ def get_individual_registrations(api, record_number, event_code=None, page=1):
# ### BEGIN ### API External Impexium Methods ### get_individual_purchases() ### # ### BEGIN ### API External Impexium Methods ### get_individual_purchases() ###
# Updated 2022-02-18 # Updated 2022-02-18
@logger_reset
def get_individual_purchases(api, record_number, product_code=None, purchased_since=None, product_category_code=None, page=1): def get_individual_purchases(api, record_number, product_code=None, purchased_since=None, product_category_code=None, page=1):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
@@ -342,38 +335,58 @@ def get_individual_purchases(api, record_number, product_code=None, purchased_si
# ### BEGIN ### API External Impexium Methods ### get_individual_custom_fields() ### # ### BEGIN ### API External Impexium Methods ### get_individual_custom_fields() ###
# Updated 2022-02-18 # Updated 2022-03-23
def get_individual_custom_fields(api, individual_id): @logger_reset
def get_individual_custom_fields(
individual_id
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
endpoint = f'/Individuals/{individual_id}/CustomFields' endpoint = f'/Individuals/{individual_id}/CustomFields'
uri = api['base_url']+endpoint uri = api['base_url']+endpoint
params = { } params = { }
resp = requests.get(url=uri, params=params, headers=api['headers']) impexium_individual_custom_field_list = None
log.debug(f'Status Code: {resp.status_code}') try_request = True
log.debug(f'Headers: {resp.headers}') max_tries = 5
# log.debug(f'Encoding: {resp.encoding}') try_count = 0
log.debug(f'JSON: {resp.json()}') while try_request and try_count <= max_tries:
# log.debug('Text:', resp.text) try_count = try_count + 1
response_data = resp.json() resp = requests.get(url=uri, params=params, headers=api['headers'])
# pp = pprint.PrettyPrinter(indent=2) log.debug(f'Status Code: {resp.status_code}')
# pp.pprint(response_data) log.debug(f'Headers: {resp.headers}')
# print('**************************') # log.debug(f'Encoding: {resp.encoding}')
# log.debug('Text:')
# log.debug(resp.text)
individual_custom_field_li = response_data if resp.status_code == 200:
log.debug(resp.json)
impexium_individual_custom_field_list = resp.json()
try_request = False
elif resp.status_code == 404:
log.info('No results returned.')
try_request = False
impexium_individual_custom_field_list = []
elif resp.status_code == 429:
log.warning('Hit rate limit. Sleeping for .1 seconds...')
time.sleep(.1)
try_request = True
impexium_individual_custom_field_list = False
else:
try_request = False
impexium_event_registration_list = False
return individual_custom_field_li return impexium_individual_custom_field_list
# ### END ### API External Impexium Methods ### get_individual_custom_fields() ### # ### END ### API External Impexium Methods ### get_individual_custom_fields() ###
# ### BEGIN ### API External Impexium Methods ### get_individual_orders_open() ### # ### BEGIN ### API External Impexium Methods ### get_individual_orders_open() ###
# Updated 2022-02-18 # Updated 2022-02-18
@logger_reset
def get_individual_orders_open(api, record_number, include_line_items=False, from_date=None, to_date=None, page=1): def get_individual_orders_open(api, record_number, include_line_items=False, from_date=None, to_date=None, page=1):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())

3
app/methods/event_badge_methods.py
View File

@@ -98,6 +98,7 @@ def get_event_badge_template_id_w_event_id(
# ### BEGIN ### API Event Badge Methods ### create_update_event_badge_obj_v4() ### # ### BEGIN ### API Event Badge Methods ### create_update_event_badge_obj_v4() ###
# Updated 2022-02-23 # Updated 2022-02-23
@logger_reset
def create_update_event_badge_obj_v4( def create_update_event_badge_obj_v4(
event_badge_dict_obj: Event_Badge_Base|dict, event_badge_dict_obj: Event_Badge_Base|dict,
event_badge_id: int|str = None, event_badge_id: int|str = None,
@@ -108,7 +109,7 @@ def create_update_event_badge_obj_v4(
fail_any: bool = False, # Fail if any thing goes wrong for sub objects fail_any: bool = False, # Fail if any thing goes wrong for sub objects
return_outline: bool = False, return_outline: bool = False,
) -> int|bool: ) -> int|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
# ### SECTION ### Secondary data validation # ### SECTION ### Secondary data validation

3
app/models/event_badge_models.py
View File

@@ -43,7 +43,8 @@ class Event_Badge_Base(BaseModel):
person_id_random: Optional[str] person_id_random: Optional[str]
person_id: Optional[int] person_id: Optional[int]
external_id: Optional[str] external_id: Optional[str] # Generated internally or externally. Needs to be stable. It should not change.
external_sys_id: Optional[str] # Generated by external system (should be stable and not change)
pronouns: Optional[str] # Preferred pronouns pronouns: Optional[str] # Preferred pronouns
informal_name: Optional[str] informal_name: Optional[str]

3
app/models/event_person_models.py
View File

@@ -56,7 +56,8 @@ class Event_Person_Base(BaseModel):
user_id_random: Optional[str] user_id_random: Optional[str]
user_id: Optional[int] user_id: Optional[int]
external_id: Optional[str] external_id: Optional[str] # Generated internally or externally. Needs to be stable. It should not change.
external_sys_id: Optional[str] # Generated by external system (should be stable and not change)
file_count: Optional[int] file_count: Optional[int]

65
app/routers/e_impexium.py
View File

@@ -10,7 +10,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.e_impexium_methods import get_custom_fields, get_events, get_event_registrants from app.methods.e_impexium_methods import get_custom_fields, get_events, get_event_registrants, get_individual_custom_fields
from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3 from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3
@@ -28,6 +28,7 @@ async def event_import_reg(
e_impexium_event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: 42_AM (2022-04); EX22_AM (2022-04); 41V_2 (2021-04) e_impexium_event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: 42_AM (2022-04); EX22_AM (2022-04); 41V_2 (2021-04)
registered_since: datetime.datetime = None, # datetime.datetime.now() + datetime.timedelta(seconds=120) registered_since: datetime.datetime = None, # datetime.datetime.now() + datetime.timedelta(seconds=120)
details: bool = True, details: bool = True,
inc_membership_fields: bool = True,
page: int = 0, # 250 per page from Impexium page: int = 0, # 250 per page from Impexium
event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: ZDzTBlevhZs (2022-04) event_id: str = Query(..., min_length=11, max_length=22), # For ISHLT: ZDzTBlevhZs (2022-04)
# Account ID For ISHLT: d8TqXqf1EOg # Account ID For ISHLT: d8TqXqf1EOg
@@ -54,17 +55,19 @@ async def event_import_reg(
for event_registrant in event_registrant_li: for event_registrant in event_registrant_li:
if loop_count > loop_limit: break if loop_count > loop_limit: break
log.info(f"Record Num: {event_registrant.get('recordNumber')}, Registration Num: {event_registrant.get('registrationNumber')}, Registrant Type Code: {event_registrant.get('registrantTypeCode')}") log.info(f"Record Num: {event_registrant.get('recordNumber')}, Registration Num: {event_registrant.get('registrationNumber')}, Registrant Type Code: {event_registrant.get('registrantTypeCode')}")
log.debug(event_registrant) # log.debug(event_registrant)
# Creating an external ID from the Impexium record number and registration number. Very explicitly converting each number to a string before joining, just in case. # Creating an external ID from the Impexium record number and registration number. Very explicitly converting each number to a string before joining, just in case.
external_id_old = str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber')) external_id_old = str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber'))
external_id_new = str(e_impexium_event_id)+':'+str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber')) external_id_new = str(e_impexium_event_id)+':'+str(event_registrant.get('recordNumber'))+':'+str(event_registrant.get('registrationNumber'))
external_sys_id = event_registrant.get('id') # Impexium UUID for person
event_person_data = {} event_person_data = {}
event_person_data['account_id'] = account_id event_person_data['account_id'] = account_id
event_person_data['event_id'] = event_id event_person_data['event_id'] = event_id
event_person_data['external_id'] = external_id_new event_person_data['external_id'] = external_id_new
event_person_data['external_id_old'] = external_id_old event_person_data['external_sys_id'] = external_sys_id
# event_person_data['external_id_old'] = external_id_old
email = None email = None
country_subdivision_code = None country_subdivision_code = None
@@ -88,6 +91,20 @@ async def event_import_reg(
country_alpha_2_code = country_data.get('twoLetterIsoCode') country_alpha_2_code = country_data.get('twoLetterIsoCode')
country = address.get('country') country = address.get('country')
degrees = None
organization_name = None
if inc_membership_fields:
individual_custom_fields_li = get_individual_custom_fields(individual_id=event_registrant.get('id'))
for individual_custom_field in individual_custom_fields_li:
if individual_custom_field.get('name') == 'degree':
degrees = individual_custom_field.get('value')
if individual_custom_field.get('name') == 'organization_name':
organization_name = individual_custom_field.get('value')
log.debug(f'Degrees: {degrees}; Organization Name: {organization_name}')
event_person_profile_data = {} event_person_profile_data = {}
event_person_profile_data['pronouns'] = event_registrant.get('gender') event_person_profile_data['pronouns'] = event_registrant.get('gender')
event_person_profile_data['informal_name'] = event_registrant.get('preferredFirstName') event_person_profile_data['informal_name'] = event_registrant.get('preferredFirstName')
@@ -98,11 +115,17 @@ async def event_import_reg(
event_person_profile_data['family_name'] = event_registrant.get('lastName') event_person_profile_data['family_name'] = event_registrant.get('lastName')
event_person_profile_data['designations'] = event_registrant.get('suffix') event_person_profile_data['designations'] = event_registrant.get('suffix')
event_person_profile_data['professional_title'] = event_registrant.get('title') if degrees:
event_person_profile_data['professional_title'] = degrees # Ideally should be degrees for ISHLT
else:
event_person_profile_data['professional_title'] = event_registrant.get('title') # Should this be None if no degrees found above???
event_person_profile_data['display_name'] = event_registrant.get('badgeName') event_person_profile_data['display_name'] = event_registrant.get('badgeName')
event_person_profile_data['affiliations'] = event_registrant.get('badgeOrganization') if organization_name:
event_person_profile_data['affiliations'] = organization_name # Ideally should be organization_name for ISHLT
else:
event_person_profile_data['affiliations'] = event_registrant.get('badgeOrganization') # Should this be None if no organization_name found above???
if email: event_person_profile_data['email'] = email if email: event_person_profile_data['email'] = email
@@ -112,7 +135,8 @@ async def event_import_reg(
event_badge_data = {} event_badge_data = {}
event_badge_data['event_id'] = event_id event_badge_data['event_id'] = event_id
event_badge_data['external_id'] = external_id_new event_badge_data['external_id'] = external_id_new
event_badge_data['external_id_old'] = external_id_old event_badge_data['external_sys_id'] = external_sys_id
# event_badge_data['external_id_old'] = external_id_old
if reg_type_code := event_registrant.get('registrantTypeCode'): if reg_type_code := event_registrant.get('registrantTypeCode'):
event_badge_data['badge_type_code'] = event_registrant.get('registrantTypeCode') # Using this as the badge_type event_badge_data['badge_type_code'] = event_registrant.get('registrantTypeCode') # Using this as the badge_type
@@ -204,11 +228,17 @@ async def event_import_reg(
event_badge_data['family_name'] = event_registrant.get('lastName') event_badge_data['family_name'] = event_registrant.get('lastName')
event_badge_data['designations'] = event_registrant.get('suffix') event_badge_data['designations'] = event_registrant.get('suffix')
event_badge_data['professional_title'] = event_registrant.get('title') if degrees:
event_badge_data['professional_title'] = degrees # Ideally should be degrees for ISHLT
else:
event_badge_data['professional_title'] = event_registrant.get('title') # Should this be None if no degrees found above???
event_badge_data['display_name'] = event_registrant.get('badgeName') event_badge_data['display_name'] = event_registrant.get('badgeName')
event_badge_data['affiliations'] = event_registrant.get('badgeOrganization') if organization_name:
event_badge_data['affiliations'] = organization_name # Ideally should be organization_name for ISHLT
else:
event_badge_data['affiliations'] = event_registrant.get('badgeOrganization') # Should this be None if no organization_name found above???
if email: event_badge_data['email'] = email if email: event_badge_data['email'] = email
@@ -370,6 +400,25 @@ async def testing(
) )
log.debug(type(event_registrant_li)) log.debug(type(event_registrant_li))
for event_registrant in event_registrant_li:
log.debug(event_registrant)
log.info(f"ID: {event_registrant.get('id')}, Registration Num: {event_registrant.get('registrationNumber')}, Registrant Name: {event_registrant.get('badgeName')}")
individual_custom_fields_li = get_individual_custom_fields(individual_id=event_registrant.get('id'))
# log.debug(individual_custom_fields_li)
degrees = None
organization_name = None
for individual_custom_field in individual_custom_fields_li:
if individual_custom_field.get('name') == 'degree':
degrees = individual_custom_field.get('value')
if individual_custom_field.get('organization_name') == 'organization_name':
organization_name = individual_custom_field.get('value')
log.debug(f'Degree: {degrees}; Organization Name: {organization_name}')
if event_registrant_li: if event_registrant_li:
return mk_resp(data=event_registrant_li, status_code=200, response=commons.response) return mk_resp(data=event_registrant_li, status_code=200, response=commons.response)
# return mk_resp(data=len(event_registrant_li), status_code=200, response=commons.response) # return mk_resp(data=len(event_registrant_li), status_code=200, response=commons.response)