Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prep for AAPOR with Confex and NCSD STD. General clean up.

Browse Source
This commit is contained in:
Scott Idem
2023-05-02 09:50:24 -04:00
parent c687ade3fa
commit c1f68522aa
6 changed files with 519 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
app/routers/e_confex.py
View File

@@ -16,7 +16,8 @@ from app.methods.event_presentation_methods import create_update_event_presentat
from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4
from app.methods.event_session_methods import create_update_event_session_obj_v4
# from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail, get_event_file_save_local, get_event_slot_data
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
from app.models.event_presentation_models import Event_Presentation_Base
from app.models.event_presenter_models import Event_Presenter_Base
@@ -40,7 +41,7 @@ async def import_event_session_list(
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
log.warning('Starting Confex event session list import...')
@@ -65,7 +66,7 @@ async def import_event_session_list(
# BEGIN: Loop through the sessions pulled from Confex
loop_count = 0
loop_limit = 200 # 200
loop_limit = 220 # 200
event_session_summary_li = []
for event_session in event_session_li:
if loop_count > loop_limit: break
@@ -73,13 +74,84 @@ async def import_event_session_list(
confex_session_id = event_session.get('id')
if confex_session_id in ('1181'): # '1110', '1100'
# log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_session_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_session_id} ***')
continue
# if confex_session_id in ('1181', '1110', '1100', '1076'): # '1110', '1100'
# # log.setLevel(logging.DEBUG)
# log.warning(f'**************** FOUND {confex_session_id} ******************')
# else:
# log.setLevel(logging.INFO)
# log.warning(f'*** FOUND {confex_session_id} ***')
# continue
# *** Part 1: *** Deal with the event_location first
# log.setLevel(logging.DEBUG)
log.debug(event_session.get('ParentList_SlotData'))
if event_session.get('ParentList_SlotData') and len(event_session.get('ParentList_SlotData')):
confex_slot_id = event_session.get('ParentList_SlotData')[0]
confex_slot_data = get_event_slot_data(confex_slot_id = confex_slot_id)
log.debug(confex_slot_data)
# event_location_external_id = confex_slot_id # This will not work because it is not unique per room.
event_location_external_id = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
event_location_code = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
event_location_name = confex_slot_data.get('Room_Name')
data = {}
data['account_id'] = account_id
data['event_id'] = event_id
# ### INSERT or UPDATE event_location
# NOTE: In the future it may make more sense to pull this location part out and or add a checked ID array so we can avoid SQL queries.
data['event_location_external_id'] = event_location_external_id
sql = f"""
SELECT *
FROM `v_event_location` AS `event_location`
WHERE event_location.event_id = :event_id
AND event_location.external_id = :event_location_external_id
LIMIT 1;
"""
event_location_data = {}
event_location_data['code'] = event_location_code
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
if event_location_name:
event_location_data['name'] = event_location_name
else:
event_location_data['name'] = 'Unknown'
event_location_data['enable'] = True
event_location_id = None
if event_location_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_location record
event_location_rec = event_location_rec_result
event_location_id = event_location_rec.get('event_location_id')
event_location_id_random = event_location_rec.get('event_location_id_random')
event_location_name = event_location_data.get('name')
log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}')
event_location_data['id'] = event_location_id
log.debug(event_location_data)
if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_up_result)
else:
log.warning(event_location_obj_up_result)
pass # Probably nothing to update
else:
log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}')
event_location_data['external_id'] = event_location_external_id # NOTE: This should never change
log.debug(event_location_data)
if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_in_result)
event_location_id = event_location_obj_in_result
else:
log.warning(event_location_obj_in_result)
continue # Something unexpected may have happened
data['event_location_id'] = event_location_id
event_session_summary = {}
event_session_summary['event_session_external_id'] = event_session.get('id')
@@ -89,27 +161,26 @@ async def import_event_session_list(
session_start_datetime_str = session_time_parts[0]+'T'+session_time_parts[1]
session_end_datetime_str = session_time_parts[0]+'T'+session_time_parts[2]
event_session_summary['event_session_start_datetime'] = session_start_datetime_str
event_session_summary['event_session_end_datetime'] = session_end_datetime_str
# event_session_summary['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# end_time???
# event_session_summary['event_session_end_datetime'] = session_end_datetime_str
event_session_summary['event_session_type'] = event_session.get('ProgramCode')
event_session_summary['event_session_type_name'] = event_session.get('ProgramTitle')
event_session_summary['event_session_subject_name'] = event_session.get('SubjectTitle')
event_session_summary['event_session_file_list'] = event_session.get('Child_MediaFiles')
event_session_summary['event_session_slot_data'] = event_session.get('ParentList_SlotData')
event_session_summary['event_location_id'] = event_location_id
event_session_data = {}
event_session_data['external_id'] = event_session.get('id')
event_session_data['name'] = event_session.get('Title')
event_session_data['start_datetime'] = session_start_datetime_str
event_session_data['end_datetime'] = session_end_datetime_str
# event_session_data['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# end_time???
event_session_data['event_location_id'] = event_location_id
# event_session_data['event_session_type'] = event_session.get('ProgramCode')
# event_session_data['event_session_type_name'] = event_session.get('ProgramTitle')
# event_session_data['event_session_subject_name'] = event_session.get('SubjectTitle')
# event_session_data['event_session_file_list'] = event_session.get('Child_MediaFiles')
# SortableTimeString - '2023-05-11_08:00:00_09:30:00'
# ParentList_Day - ['Day/2023-05-11']
# Child_MediaFiles
@@ -187,7 +258,8 @@ async def import_event_session_list(
if confex_session_detail:
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.DEBUG)
log.info(f'Session Detail: {confex_session_detail}')
log.info(f'Got session detail for Confex session {confex_session_id}')
log.debug(f'Session Detail: {confex_session_detail}')
# Presentation List
event_session_summary['event_presentation_list'] = []
@@ -197,13 +269,13 @@ async def import_event_session_list(
# Presentation Information
confex_presentation_id = confex_presentation_paper
if confex_presentation_id in ('Paper/1928'): # '', ''
log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_presentation_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_presentation_id} ***')
continue
# if confex_presentation_id in ('Paper/1928', 'Paper/1770', 'Paper/1214'): # '', ''
# log.setLevel(logging.DEBUG)
# log.warning(f'**************** FOUND {confex_presentation_id} ******************')
# else:
# log.setLevel(logging.INFO)
# log.warning(f'*** FOUND {confex_presentation_id} ***')
# continue
event_presentation_data = {}
event_presentation_data['external_id'] = confex_presentation_id
@@ -287,34 +359,8 @@ async def import_event_session_list(
# Presentation File List (not Presenter specific???)
log.info('Get presentation files for a presenter...')
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
event_presentation_data['event_file_list'] = []
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(confex_vendor_file)
confex_file_id = confex_vendor_file
log.info('Confex: Getting file detail...')
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
event_file_data = {}
if confex_file_detail:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'File Detail: {confex_file_detail}')
event_file_data['filename'] = confex_file_detail.get('FileName')
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
event_file_data['ext'] = confex_file_detail.get('Ext')
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
event_file_data['url'] = confex_file_detail.get('URL')
event_file_data['created_on'] = confex_file_detail.get('_originated')
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
event_presentation_data['event_file_list'].append(event_file_data)
# Presenter Information
submitter_full_name = confex_presentation_detail.get('SubmitterFullName')
@@ -393,10 +439,10 @@ async def import_event_session_list(
try:
event_presenter_obj = Event_Presenter_Base(**event_presenter_data_sql)
log.warning(event_presenter_obj)
except ValidationError as e:
log.error(e.json())
return False
log.debug(event_presenter_obj)
event_presenter_id = None
if event_presenter_result := sql_select(sql=sql_select_event_presenter, data=event_presenter_qry):
@@ -439,6 +485,44 @@ async def import_event_session_list(
break
# return False
# Presentation File List for Presenter
log.info('Get presentation files for a presenter...')
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
event_presentation_data['confex_file_list'] = []
event_presentation_data['event_file_list'] = []
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(confex_vendor_file)
confex_file_id = confex_vendor_file
log.info('Confex: Getting file detail...')
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
event_file_data = {}
file_info = {} # For locally saved file info below
if confex_file_detail:
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'File Detail: {confex_file_detail}')
event_file_data['filename'] = confex_file_detail.get('FileName')
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
event_file_data['extension'] = confex_file_detail.get('Ext')
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
event_file_data['url'] = confex_file_detail.get('URL')
event_file_data['created_on'] = confex_file_detail.get('_originated')
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
file_info = await get_event_file_save_local(
url = confex_file_detail.get('URL'),
confex_file_info = event_file_data,
account_id = account_id,
link_to_type = 'event_presenter',
link_to_id = event_presenter_id,
)
event_presentation_data['confex_file_list'].append(event_file_data)
event_presentation_data['event_file_list'].append(file_info)
# event_presenter_data['event_presenter_id'] = event_presenter_id # _random
if len(event_presentation_data['event_presenter_list']):
event_presentation_data['event_presenter_list'][0]['event_presenter_id'] = event_presenter_id # _random