Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prep for AAPOR with Confex and NCSD STD. General clean up.

Browse Source
This commit is contained in:
Scott Idem
2023-05-02 09:50:24 -04:00
parent c687ade3fa
commit c1f68522aa
6 changed files with 519 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

186
app/routers/e_confex.py
View File

@@ -16,7 +16,8 @@ from app.methods.event_presentation_methods import create_update_event_presentat
from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4
from app.methods.event_session_methods import create_update_event_session_obj_v4
# from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail, get_event_file_save_local, get_event_slot_data
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
from app.models.event_presentation_models import Event_Presentation_Base
from app.models.event_presenter_models import Event_Presenter_Base
@@ -40,7 +41,7 @@ async def import_event_session_list(
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
log.warning('Starting Confex event session list import...')
@@ -65,7 +66,7 @@ async def import_event_session_list(
# BEGIN: Loop through the sessions pulled from Confex
loop_count = 0
loop_limit = 200 # 200
loop_limit = 220 # 200
event_session_summary_li = []
for event_session in event_session_li:
if loop_count > loop_limit: break
@@ -73,13 +74,84 @@ async def import_event_session_list(
confex_session_id = event_session.get('id')
if confex_session_id in ('1181'): # '1110', '1100'
# log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_session_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_session_id} ***')
continue
# if confex_session_id in ('1181', '1110', '1100', '1076'): # '1110', '1100'
# # log.setLevel(logging.DEBUG)
# log.warning(f'**************** FOUND {confex_session_id} ******************')
# else:
# log.setLevel(logging.INFO)
# log.warning(f'*** FOUND {confex_session_id} ***')
# continue
# *** Part 1: *** Deal with the event_location first
# log.setLevel(logging.DEBUG)
log.debug(event_session.get('ParentList_SlotData'))
if event_session.get('ParentList_SlotData') and len(event_session.get('ParentList_SlotData')):
confex_slot_id = event_session.get('ParentList_SlotData')[0]
confex_slot_data = get_event_slot_data(confex_slot_id = confex_slot_id)
log.debug(confex_slot_data)
# event_location_external_id = confex_slot_id # This will not work because it is not unique per room.
event_location_external_id = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
event_location_code = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
event_location_name = confex_slot_data.get('Room_Name')
data = {}
data['account_id'] = account_id
data['event_id'] = event_id
# ### INSERT or UPDATE event_location
# NOTE: In the future it may make more sense to pull this location part out and or add a checked ID array so we can avoid SQL queries.
data['event_location_external_id'] = event_location_external_id
sql = f"""
SELECT *
FROM `v_event_location` AS `event_location`
WHERE event_location.event_id = :event_id
AND event_location.external_id = :event_location_external_id
LIMIT 1;
"""
event_location_data = {}
event_location_data['code'] = event_location_code
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
if event_location_name:
event_location_data['name'] = event_location_name
else:
event_location_data['name'] = 'Unknown'
event_location_data['enable'] = True
event_location_id = None
if event_location_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_location record
event_location_rec = event_location_rec_result
event_location_id = event_location_rec.get('event_location_id')
event_location_id_random = event_location_rec.get('event_location_id_random')
event_location_name = event_location_data.get('name')
log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}')
event_location_data['id'] = event_location_id
log.debug(event_location_data)
if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_up_result)
else:
log.warning(event_location_obj_up_result)
pass # Probably nothing to update
else:
log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}')
event_location_data['external_id'] = event_location_external_id # NOTE: This should never change
log.debug(event_location_data)
if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_in_result)
event_location_id = event_location_obj_in_result
else:
log.warning(event_location_obj_in_result)
continue # Something unexpected may have happened
data['event_location_id'] = event_location_id
event_session_summary = {}
event_session_summary['event_session_external_id'] = event_session.get('id')
@@ -89,27 +161,26 @@ async def import_event_session_list(
session_start_datetime_str = session_time_parts[0]+'T'+session_time_parts[1]
session_end_datetime_str = session_time_parts[0]+'T'+session_time_parts[2]
event_session_summary['event_session_start_datetime'] = session_start_datetime_str
event_session_summary['event_session_end_datetime'] = session_end_datetime_str
# event_session_summary['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# end_time???
# event_session_summary['event_session_end_datetime'] = session_end_datetime_str
event_session_summary['event_session_type'] = event_session.get('ProgramCode')
event_session_summary['event_session_type_name'] = event_session.get('ProgramTitle')
event_session_summary['event_session_subject_name'] = event_session.get('SubjectTitle')
event_session_summary['event_session_file_list'] = event_session.get('Child_MediaFiles')
event_session_summary['event_session_slot_data'] = event_session.get('ParentList_SlotData')
event_session_summary['event_location_id'] = event_location_id
event_session_data = {}
event_session_data['external_id'] = event_session.get('id')
event_session_data['name'] = event_session.get('Title')
event_session_data['start_datetime'] = session_start_datetime_str
event_session_data['end_datetime'] = session_end_datetime_str
# event_session_data['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
# end_time???
event_session_data['event_location_id'] = event_location_id
# event_session_data['event_session_type'] = event_session.get('ProgramCode')
# event_session_data['event_session_type_name'] = event_session.get('ProgramTitle')
# event_session_data['event_session_subject_name'] = event_session.get('SubjectTitle')
# event_session_data['event_session_file_list'] = event_session.get('Child_MediaFiles')
# SortableTimeString - '2023-05-11_08:00:00_09:30:00'
# ParentList_Day - ['Day/2023-05-11']
# Child_MediaFiles
@@ -187,7 +258,8 @@ async def import_event_session_list(
if confex_session_detail:
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
# log.setLevel(logging.DEBUG)
log.info(f'Session Detail: {confex_session_detail}')
log.info(f'Got session detail for Confex session {confex_session_id}')
log.debug(f'Session Detail: {confex_session_detail}')
# Presentation List
event_session_summary['event_presentation_list'] = []
@@ -197,13 +269,13 @@ async def import_event_session_list(
# Presentation Information
confex_presentation_id = confex_presentation_paper
if confex_presentation_id in ('Paper/1928'): # '', ''
log.setLevel(logging.DEBUG)
log.warning(f'**************** FOUND {confex_presentation_id} ******************')
else:
log.setLevel(logging.INFO)
log.warning(f'*** FOUND {confex_presentation_id} ***')
continue
# if confex_presentation_id in ('Paper/1928', 'Paper/1770', 'Paper/1214'): # '', ''
# log.setLevel(logging.DEBUG)
# log.warning(f'**************** FOUND {confex_presentation_id} ******************')
# else:
# log.setLevel(logging.INFO)
# log.warning(f'*** FOUND {confex_presentation_id} ***')
# continue
event_presentation_data = {}
event_presentation_data['external_id'] = confex_presentation_id
@@ -287,34 +359,8 @@ async def import_event_session_list(
# Presentation File List (not Presenter specific???)
log.info('Get presentation files for a presenter...')
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
event_presentation_data['event_file_list'] = []
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(confex_vendor_file)
confex_file_id = confex_vendor_file
log.info('Confex: Getting file detail...')
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
event_file_data = {}
if confex_file_detail:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'File Detail: {confex_file_detail}')
event_file_data['filename'] = confex_file_detail.get('FileName')
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
event_file_data['ext'] = confex_file_detail.get('Ext')
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
event_file_data['url'] = confex_file_detail.get('URL')
event_file_data['created_on'] = confex_file_detail.get('_originated')
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
event_presentation_data['event_file_list'].append(event_file_data)
# Presenter Information
submitter_full_name = confex_presentation_detail.get('SubmitterFullName')
@@ -393,10 +439,10 @@ async def import_event_session_list(
try:
event_presenter_obj = Event_Presenter_Base(**event_presenter_data_sql)
log.warning(event_presenter_obj)
except ValidationError as e:
log.error(e.json())
return False
log.debug(event_presenter_obj)
event_presenter_id = None
if event_presenter_result := sql_select(sql=sql_select_event_presenter, data=event_presenter_qry):
@@ -439,6 +485,44 @@ async def import_event_session_list(
break
# return False
# Presentation File List for Presenter
log.info('Get presentation files for a presenter...')
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
event_presentation_data['confex_file_list'] = []
event_presentation_data['event_file_list'] = []
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(confex_vendor_file)
confex_file_id = confex_vendor_file
log.info('Confex: Getting file detail...')
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
event_file_data = {}
file_info = {} # For locally saved file info below
if confex_file_detail:
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(f'File Detail: {confex_file_detail}')
event_file_data['filename'] = confex_file_detail.get('FileName')
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
event_file_data['extension'] = confex_file_detail.get('Ext')
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
event_file_data['url'] = confex_file_detail.get('URL')
event_file_data['created_on'] = confex_file_detail.get('_originated')
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
file_info = await get_event_file_save_local(
url = confex_file_detail.get('URL'),
confex_file_info = event_file_data,
account_id = account_id,
link_to_type = 'event_presenter',
link_to_id = event_presenter_id,
)
event_presentation_data['confex_file_list'].append(event_file_data)
event_presentation_data['event_file_list'].append(file_info)
# event_presenter_data['event_presenter_id'] = event_presenter_id # _random
if len(event_presentation_data['event_presenter_list']):
event_presentation_data['event_presenter_list'][0]['event_presenter_id'] = event_presenter_id # _random

44
app/routers/event_importing.py
View File

@@ -331,7 +331,7 @@ async def event_importing_create_update_w_external_id(
# ### BEGIN ### Event Importing ### event_importing_program_data() ###
# Based on the program import template the clients are given.
# Create and update locations, sessions, presentations, and presenters as needed.
# Updated 2022-10-07
# Updated 2023-05-01
@router.post('/event/{event_id}/importing/program_data', response_model=Resp_Body_Base)
async def event_importing_program_data(
event_id: str = Query(..., min_length=11, max_length=22),
@@ -410,6 +410,7 @@ async def event_importing_program_data(
'location_external_id': str, 'location_code': str, 'location_sort': int,
'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int,
'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int,
'session_external_id': str, 'session_code': str, 'session_sort': int,
@@ -441,19 +442,22 @@ async def event_importing_program_data(
'Affiliations': 'presenter_presenter_affiliations',
'Company': 'presenter_presenter_affiliations',
'email': 'presenter_email',
'Email': 'presenter_email',
'Email Address': 'presenter_email',
'CC Email Address': 'presenter_cc_email',
'location_title': 'location_name',
'Location Code': 'location_code',
'session_location': 'location_code',
'Session Location': 'location_code',
'location_title': 'location_name',
'Location Name': 'location_name',
'session_location': 'location_name',
'Session Location': 'location_name',
# 'session_location': 'location_name',
'session_title': 'session_name',
'Session Code': 'session_code',
'Session Name': 'session_name',
'Session Hide': 'hide_session',
'presentation_title': 'presentation_name',
'Presentation Code': 'presentation_code',
@@ -462,6 +466,9 @@ async def event_importing_program_data(
'Presenter Code': 'presenter_code',
'Presenter Number': 'presenter_number', # for sorting
'Presenter Name': 'presenter_name',
'Presenter Notes': 'presenter_notes',
'Notes': 'presenter_notes',
},
inplace = True)
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -488,6 +495,7 @@ async def event_importing_program_data(
else:
log.warning('No location external ID or code was found. Creating one with the name...')
event_location_name = record.get('location_name', '').strip()
# event_location_code = event_location_name
event_location_external_id = f'{event_location_name}' # .strip()
log.debug(f'Event Location External ID: {event_location_external_id}')
@@ -547,7 +555,10 @@ async def event_importing_program_data(
event_location_data = {}
event_location_data['code'] = record.get('location_code')
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_location_data['name'] = record.get('location_name')
if record.get('location_name'):
event_location_data['name'] = record.get('location_name')
elif record.get('location_code'):
event_location_data['name'] = record.get('location_code')
event_location_data['enable'] = True
event_location_id = None
@@ -613,6 +624,9 @@ async def event_importing_program_data(
event_session_data['sort'] = record.get('session_sort')
if record.get('hide_session') in (True, 1, 'yes', 'Yes', 'YES'):
event_session_data['hide'] = True
event_session_data['enable'] = True
event_session_data['notes'] = record.get('session_notes')
@@ -670,15 +684,19 @@ async def event_importing_program_data(
event_presentation_data['name'] = record.get('presentation_name')
event_presentation_data['description'] = record.get('presentation_description')
event_presentation_data['start_datetime'] = record.get('presentation_start_datetime')
# event_presentation_start_datetime = record.get('presentation_start_date', '') + ' ' + record.get('presentation_start_time', '')
# event_presentation_data['start_datetime'] = event_presentation_start_datetime
data['presentation_start_datetime'] = event_presentation_data['start_datetime']
if record.get('presentation_start_datetime'):
event_presentation_data['start_datetime'] = record.get('presentation_start_datetime')
data['presentation_start_datetime'] = event_presentation_data['start_datetime']
else:
event_presentation_data['start_datetime'] = None
data['presentation_start_datetime'] = None
event_presentation_data['end_datetime'] = record.get('presentation_end_datetime')
# event_presentation_end_datetime = record.get('presentation_end_date', '') + ' ' + record.get('presentation_end_time', '')
# event_presentation_data['end_datetime'] = event_presentation_end_datetime
data['presentation_end_datetime'] = event_presentation_data['end_datetime']
if record.get('presentation_end_datetime'):
event_presentation_data['end_datetime'] = record.get('presentation_end_datetime')
data['presentation_end_datetime'] = event_presentation_data['end_datetime']
else:
event_presentation_data['end_datetime'] = None
data['presentation_end_datetime'] = None
event_presentation_data['sort'] = record.get('presentation_sort')

0
app/routers/hosted_file_help.py
View File