Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prep for AAPOR with Confex. General clean up.

Browse Source
This commit is contained in:
Scott Idem
2023-04-30 19:21:19 -04:00
parent 10b8e8ca7a
commit c687ade3fa
5 changed files with 338 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
app/methods/e_confex_methods.py
View File

@@ -159,7 +159,7 @@ def get_event_presentation_detail(
confex_session_id: str,
confex_presentation_id: str, # similar to 'Paper/99999'
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if result := authenticate():
@@ -167,7 +167,8 @@ def get_event_presentation_detail(
# else:
# return False
endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}'
# endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}'
endpoint = f'/{confex_presentation_id}'
uri = api['base_url']+endpoint
params = {}
@@ -200,7 +201,7 @@ def get_event_presentation_detail(
try_request = False
elif resp.status_code == 404:
log.info('No results returned (status 404)')
log.warning('No results returned (status 404)')
try_request = False
confex_presentation_detail = None
elif resp.status_code == 429:
@@ -209,7 +210,7 @@ def get_event_presentation_detail(
try_request = True
confex_presentation_detail = False
else:
log.info('Not trying again')
log.warning('Not trying again')
try_request = False
confex_presentation_detail = False
@@ -290,4 +291,74 @@ def get_event_presenter_detail(
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
return confex_presenter_detail
# ### END ### API External Confex Methods ### get_event_presenter_detail() ###
# ### END ### API External Confex Methods ### get_event_presenter_detail() ###
# ### BEGIN ### API External Confex Methods ### get_event_file_detail() ###
# Updated 2023-04-30
@logger_reset
def get_event_file_detail(
confex_file_id: str, # similar to 'FileMap/Paper1928_Presentation2'
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if result := authenticate():
# log.debug(result)
# else:
# return False
endpoint = f'/{confex_file_id}'
uri = api['base_url']+endpoint
params = {}
confex_file_detail = None
try_request = True
max_tries = 5
try_count = 0
while try_request and try_count <= max_tries:
try_count = try_count + 1
resp = requests.get(url=uri, params=params, headers=api['headers'])
log.debug(f'Status Code: {resp.status_code}')
log.debug(f'Headers: {resp.headers}')
# log.debug(f'Encoding: {resp.encoding}')
# log.debug('Text:')
# log.debug(resp.text)
# log.debug(resp.json())
if resp.status_code == 200:
log.info('Status 200')
log.debug(resp.json())
confex_file_detail_raw = resp.json() # .get('data').get('dataList')[0]
# log.debug(confex_file_detail_raw)
confex_file_detail = confex_file_detail_raw
log.debug(confex_file_detail)
try_request = False
elif resp.status_code == 404:
log.warning('No results returned (status 404)')
try_request = False
confex_file_detail = None
elif resp.status_code == 429:
log.warning('Hit rate limit. Sleeping for .1 seconds...')
time.sleep(.1)
try_request = True
confex_file_detail = False
else:
log.warning('Not trying again')
try_request = False
confex_file_detail = False
log.warning('Something may have gone wrong during the request.')
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Impexium on the next request.')
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
return confex_file_detail
# ### END ### API External Confex Methods ### get_event_file_detail() ###

2
app/methods/event_file_methods.py
View File

@@ -187,7 +187,7 @@ def get_event_file_rec_list(
event_file_rec_li = event_file_rec_li_result
else:
event_file_rec_li = []
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(event_file_rec_li_result)
log.debug(type(event_file_rec_li))
log.debug(len(event_file_rec_li))

2
app/methods/event_presentation_methods.py
View File

@@ -361,7 +361,7 @@ def create_update_event_presentation_obj_v4(
fail_any: bool = False, # Fail if any thing goes wrong for sub objects
return_outline: bool = False,
) -> int|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
log.info('Checking requirements...')

4
app/methods/event_session_methods.py
View File

@@ -57,7 +57,7 @@ def load_event_session_obj(
exclude_unset: bool = True,
model_as_dict: bool = False,
) -> Event_Session_Base|bool:
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if event_session_id := redis_lookup_id_random(record_id_random=event_session_id, table_name='event_session'): pass
@@ -293,7 +293,7 @@ def load_event_session_obj(
log.debug(event_presenter_cat_rec_result.get('event_presenter_names', None))
event_session_obj.event_presenter_cat = event_presenter_cat_rec_result.get('event_presenter_names', None)
else:
log.warning(event_presenter_cat_rec_result)
log.info(event_presenter_cat_rec_result)
event_session_obj.event_presenter_cat = None
if inc_event_presenter_list: pass

289
app/routers/e_confex.py
View File

@@ -1,7 +1,7 @@
import datetime, json, pytz, secrets, time
import pandas, xlrd # qrcode
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Response, status
from pydantic import BaseModel, EmailStr, Field
from pydantic import BaseModel, EmailStr, Field, ValidationError
from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging, secure_hash_string, common_route_params, Common_Route_Params
@@ -12,10 +12,14 @@ from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_
from app.methods.e_impexium_methods import get_custom_fields, get_events, get_event_registrants, get_individual_custom_fields, get_individual_profile
from app.methods.event_presentation_methods import create_update_event_presentation_obj_v4
from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4
from app.methods.event_session_methods import create_update_event_session_obj_v4
# from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail
from app.models.event_presentation_models import Event_Presentation_Base
from app.models.event_presenter_models import Event_Presenter_Base
from app.models.event_session_models import Event_Session_Base
from app.models.response_models import Resp_Body_Base, mk_resp
@@ -69,6 +73,14 @@ async def import_event_session_list(
confex_session_id = event_session.get('id')
if confex_session_id in ('1181'): # '1110', '1100'
# log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_session_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_session_id} ***')
continue
event_session_summary = {}
event_session_summary['event_session_external_id'] = event_session.get('id')
event_session_summary['event_session_name'] = event_session.get('Title')
@@ -78,7 +90,7 @@ async def import_event_session_list(
session_end_datetime_str = session_time_parts[0]+'T'+session_time_parts[2]
event_session_summary['event_session_start_datetime'] = session_start_datetime_str
event_session_summary['event_session_end_datetime'] = session_end_datetime_str
event_session_summary['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# event_session_summary['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# end_time???
event_session_summary['event_session_type'] = event_session.get('ProgramCode')
event_session_summary['event_session_type_name'] = event_session.get('ProgramTitle')
@@ -114,7 +126,7 @@ async def import_event_session_list(
event_session_qry['event_id'] = event_id
event_session_qry['external_id'] = confex_session_id
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(event_session_qry)
sql_select_event_session = f"""
@@ -128,7 +140,7 @@ async def import_event_session_list(
try:
event_session_obj = Event_Session_Base(**event_session_data)
log.warning(event_session_obj)
log.debug(event_session_obj)
except ValidationError as e:
log.error(e.json())
return False
@@ -173,8 +185,9 @@ async def import_event_session_list(
event_session_data['event_presentation_list'] = None # Confex "Paper"
if confex_session_detail:
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'Session Detail: {confex_session_detail}')
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.DEBUG)
log.info(f'Session Detail: {confex_session_detail}')
# Presentation List
event_session_summary['event_presentation_list'] = []
@@ -183,55 +196,252 @@ async def import_event_session_list(
for confex_presentation_paper in confex_session_detail.get('ChildList_Paper'):
# Presentation Information
confex_presentation_id = confex_presentation_paper
if confex_presentation_id in ('Paper/1928'): # '', ''
log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_presentation_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_presentation_id} ***')
continue
event_presentation_data = {}
event_presentation_data['external_id'] = confex_presentation_id
log.info('Confex: Getting presentation detail...')
confex_presentation_detail = get_event_presentation_detail(confex_session_id = confex_session_id, confex_presentation_id = confex_presentation_id)
if confex_presentation_detail:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'Presentation Detail: {confex_presentation_detail}')
event_presentation_data['event_presentation_name'] = confex_presentation_detail.get('Title')
event_presentation_data['event_presentation_start_end_datetime'] = confex_presentation_detail.get('SortableTimeString')
event_presentation_data['name'] = confex_presentation_detail.get('Title')
presentation_time_parts = confex_presentation_detail.get('SortableTimeString').split('_')
presentation_start_datetime_str = presentation_time_parts[0]+'T'+presentation_time_parts[1]
presentation_end_datetime_str = presentation_time_parts[0]+'T'+presentation_time_parts[2]
event_presentation_data['start_datetime'] = presentation_start_datetime_str
event_presentation_data['end_datetime'] = presentation_end_datetime_str
# event_presentation_data['event_presentation_start_end_datetime'] = confex_presentation_detail.get('SortableTimeString')
event_presentation_qry = {}
event_presentation_qry['event_session_id'] = event_session_id
event_presentation_qry['external_id'] = confex_presentation_id
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(event_presentation_qry)
sql_select_event_presentation = f"""
SELECT id AS event_presentation_id, id_random AS event_presentation_id_random, external_id AS event_presentation_external_id
FROM `event_presentation` AS `event_presentation`
WHERE event_presentation.event_session_id = :event_session_id
AND event_presentation.external_id = :external_id
/*LIMIT 1*/;
"""
log.debug(sql_select_event_presentation)
try:
event_presentation_obj = Event_Presentation_Base(**event_presentation_data)
log.warning(event_presentation_obj)
except ValidationError as e:
log.error(e.json())
return False
if event_presentation_result := sql_select(sql=sql_select_event_presentation, data=event_presentation_qry):
if isinstance(event_presentation_result, list):
log.error(f'Found more than one Event Presentation with the same External ID. Count: {len(event_presentation_result)}')
break
# return False
else:
event_presentation_id = event_presentation_result.get('event_presentation_id')
log.info(f'Found Event Presentation. Updating existing... Event Presentation ID: {event_presentation_id}')
if create_event_presentation_obj_result := create_update_event_presentation_obj_v4(
event_presentation_dict_obj = event_presentation_obj,
event_presentation_id = event_presentation_id,
event_id = event_id,
event_session_id = event_session_id,
):
event_presentation_id = create_event_presentation_obj_result
log.warning(f'Event Presentation updated. Event Presentation ID: {event_presentation_id}')
else:
log.warning(f'Event Presentation not updated. Event Presentation ID: {event_presentation_id}')
log.debug(create_event_presentation_obj_result)
break
# return False
else:
log.info('No Event Presentation found. Creating new...')
if create_event_presentation_obj_result := create_update_event_presentation_obj_v4(
event_presentation_dict_obj = event_presentation_obj,
event_id = event_id,
event_session_id = event_session_id,
):
event_presentation_id = create_event_presentation_obj_result
log.warning(f'Event Presentation created. Event Presentation ID: {event_presentation_id}')
else:
log.warning(f'Event Presentation not created.')
log.debug(create_event_presentation_obj_result)
break
# return False
event_presentation_data['event_presentation_id'] = event_presentation_id # _random
# Presentation File List (not Presenter specific???)
log.info('Get presentation files for a presenter...')
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
event_presentation_data['event_file_list'] = []
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(confex_vendor_file)
confex_file_id = confex_vendor_file
event_file = {}
event_file['filename'] = confex_vendor_file.get('Filename')
event_file['ext'] = confex_vendor_file.get('Ext')
event_file['size'] = confex_vendor_file.get('Size') # in bytes?
event_file['file_purpose'] = confex_vendor_file.get('Table') # 'Handout', 'Presentation'
event_file['url'] = confex_vendor_file.get('URL')
event_presentation_data['event_file_list'].append(event_file)
log.info('Confex: Getting file detail...')
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
event_file_data = {}
if confex_file_detail:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'File Detail: {confex_file_detail}')
event_file_data['filename'] = confex_file_detail.get('FileName')
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
event_file_data['ext'] = confex_file_detail.get('Ext')
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
event_file_data['url'] = confex_file_detail.get('URL')
event_file_data['created_on'] = confex_file_detail.get('_originated')
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
event_presentation_data['event_file_list'].append(event_file_data)
# Presenter Information
event_presentation_data['event_presenter_full_name'] = confex_presentation_detail.get('SubmitterFullName')
submitter_full_name = confex_presentation_detail.get('SubmitterFullName')
log.info(f'Event Session/Presenter Submitter: {submitter_full_name}')
event_presentation_data['event_session_submitter_full_name'] = submitter_full_name
presenter_full_name = parse_name(confex_presentation_detail.get('TopDisplay'))
log.info(f'Event Presenter Full Name: {presenter_full_name}')
event_presentation_data['event_presenter_full_name'] = presenter_full_name
# if presenter_full_name:
# else:
# event_presentation_data['event_presenter_full_name'] = None
# Presenter List
event_presentation_data['event_presenter_list'] = []
for confex_presenter_person in confex_presentation_detail.get('ChildList_Person'):
confex_presenter_id = confex_presenter_person
event_presenter_data = {}
event_presenter_data['external_id'] = confex_presenter_id
if len(confex_presentation_detail.get('ChildList_Person')):
for confex_presenter_person in confex_presentation_detail.get('ChildList_Person'):
confex_presenter_id = confex_presenter_person
event_presenter_summary = {}
event_presenter_summary['external_id'] = confex_presenter_id
event_presenter_summary['full_name'] = confex_presentation_detail.get('SubmitterFullName')
confex_presenter_detail = get_event_presenter_detail(confex_session_id = confex_session_id, confex_presentation_id = confex_presentation_id, confex_presenter_id = confex_presenter_id)
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
if confex_presenter_detail:
log.debug(f'Presenter Detail: {confex_presenter_detail}')
confex_presenter_detail = get_event_presenter_detail(confex_session_id = confex_session_id, confex_presentation_id = confex_presentation_id, confex_presenter_id = confex_presenter_id)
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
if confex_presenter_detail:
log.debug(f'Presenter Detail: {confex_presenter_detail}')
event_presenter_data['event_presenter_given_name'] = confex_presenter_detail.get('FirstName')
event_presenter_data['event_presenter_middle_name'] = confex_presenter_detail.get('MiddleName')
event_presenter_data['event_presenter_family_name'] = confex_presenter_detail.get('LastName')
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
event_presentation_data['event_presenter_list'].append(event_presenter_data)
event_presenter_summary['given_name'] = confex_presenter_detail.get('FirstName')
event_presenter_summary['middle_name'] = confex_presenter_detail.get('MiddleName')
event_presenter_summary['family_name'] = confex_presenter_detail.get('LastName')
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
event_presentation_data['event_presenter_list'].append(event_presenter_summary)
event_presenter_data_sql = {}
if presenter_full_name:
event_presenter_data_sql['full_name'] = presenter_full_name
event_presenter_data_sql['given_name'] = presenter_full_name
else:
event_presenter_data_sql['full_name'] = submitter_full_name
event_presenter_data_sql['given_name'] = submitter_full_name
event_presenter_qry = {}
event_presenter_qry['event_session_id'] = event_session_id
event_presenter_qry['event_presentation_id'] = event_presentation_id
if len(event_presentation_data['event_presenter_list']):
event_presenter_qry['external_id'] = event_presentation_data['event_presenter_list'][0]['external_id']
event_presenter_data_sql['external_id'] = event_presentation_data['event_presenter_list'][0]['external_id']
event_presenter_data_sql['given_name'] = event_presentation_data['event_presenter_list'][0]['given_name']
event_presenter_data_sql['middle_name'] = event_presentation_data['event_presenter_list'][0]['middle_name']
event_presenter_data_sql['family_name'] = event_presentation_data['event_presenter_list'][0]['family_name']
else:
event_presenter_qry['external_id'] = None
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(event_presenter_qry)
if event_presenter_qry['external_id']:
sql_select_event_presenter = f"""
SELECT id AS event_presenter_id, id_random AS event_presenter_id_random, external_id AS event_presenter_external_id
FROM `event_presenter` AS `event_presenter`
WHERE event_presenter.event_presentation_id = :event_presentation_id
AND event_presenter.external_id = :external_id
/*LIMIT 1*/;
"""
else:
sql_select_event_presenter = f"""
SELECT id AS event_presenter_id, id_random AS event_presenter_id_random, external_id AS event_presenter_external_id
FROM `event_presenter` AS `event_presenter`
WHERE event_presenter.event_presentation_id = :event_presentation_id
AND event_presenter.external_id IS NULL
/*LIMIT 1*/;
"""
log.debug(sql_select_event_presenter)
try:
event_presenter_obj = Event_Presenter_Base(**event_presenter_data_sql)
log.warning(event_presenter_obj)
except ValidationError as e:
log.error(e.json())
return False
event_presenter_id = None
if event_presenter_result := sql_select(sql=sql_select_event_presenter, data=event_presenter_qry):
if isinstance(event_presenter_result, list):
log.error(f'Found more than one Event Presenter with the same External ID. Count: {len(event_presenter_result)}')
break
# return False
else:
event_presenter_id = event_presenter_result.get('event_presenter_id')
log.info(f'Found Event Presenter. Updating existing... Event Presenter ID: {event_presenter_id}')
if create_event_presenter_obj_result := create_update_event_presenter_obj_v4(
event_presenter_dict_obj = event_presenter_obj,
event_presenter_id = event_presenter_id,
event_id = event_id,
event_session_id = event_session_id,
event_presentation_id = event_presentation_id,
):
event_presenter_id = create_event_presenter_obj_result
log.warning(f'Event Presenter updated. Event Presenter ID: {event_presenter_id}')
else:
log.warning(f'Event Presenter not updated. Event Presenter ID: {event_presenter_id}')
log.debug(create_event_presenter_obj_result)
break
# return False
else:
log.info('No Event Presenter found. Creating new...')
if create_event_presenter_obj_result := create_update_event_presenter_obj_v4(
event_presenter_dict_obj = event_presenter_obj,
event_id = event_id,
event_session_id = event_session_id,
event_presentation_id = event_presentation_id,
):
event_presenter_id = create_event_presenter_obj_result
log.warning(f'Event Presenter created. Event Presenter ID: {event_presenter_id}')
else:
log.warning(f'Event Presenter not created.')
log.debug(create_event_presenter_obj_result)
break
# return False
# event_presenter_data['event_presenter_id'] = event_presenter_id # _random
if len(event_presentation_data['event_presenter_list']):
event_presentation_data['event_presenter_list'][0]['event_presenter_id'] = event_presenter_id # _random
event_session_summary['event_presentation_list'].append(event_presentation_data)
@@ -247,4 +457,21 @@ async def import_event_session_list(
loop_count = loop_count + 1
# if confex_session_id == '1110':
# log.setLevel(logging.DEBUG)
# log.warning('**************** FOUND 1110 ******************')
# break
return mk_resp(data=event_session_summary_li, status_code=200, response=commons.response) # Partially implemented...
def parse_name(string):
start_tag = '<b>'
end_tag = '</b>'
start_index = string.find(start_tag)
end_index = string.find(end_tag)
if start_index != -1 and end_index != -1:
return string[start_index + len(start_tag):end_index]
else:
return None