Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates to the event program data import script.

Browse Source
This commit is contained in:
Scott Idem
2024-08-15 09:39:15 -04:00
parent de35856749
commit 2ef883984d
1 changed files with 332 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

635
app/routers/event_importing.py
View File

@@ -28,304 +28,305 @@ from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter() router = APIRouter()
# No longer needed? 2024-08-15
# Based on the program import template the clients are given. # Based on the program import template the clients are given.
# Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed.
# Updated 2021-10-19 # Updated 2021-10-19
@router.get('/event/importing/create_update_w_external_id', response_model=Resp_Body_Base) # @router.get('/event/importing/create_update_w_external_id', response_model=Resp_Body_Base)
async def event_importing_create_update_w_external_id( # async def event_importing_create_update_w_external_id(
response: Response = Response, # response: Response = Response,
x_account_id: str = Header(...), # x_account_id: str = Header(...),
): # ):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) # log.debug(locals())
allow_inserts = False # allow_inserts = False
allow_updates = False # allow_updates = False
account_id = 2 # CMSC 2 8Gfxbxr19Nw # account_id = 2 # CMSC 2 8Gfxbxr19Nw
event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw # event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw
full_file_path = 'admin/temp/import_event_program_external_id.csv' # full_file_path = 'admin/temp/import_event_program_external_id.csv'
df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str}) # df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str})
log.debug(df) # log.debug(df)
df_dict = df.to_dict(orient='records') # df_dict = df.to_dict(orient='records')
event_session_data_li = [] # event_session_data_li = []
for record in df_dict: # for record in df_dict:
if event_location_external_id := record.get('event_location_external_id', None): pass # if event_location_external_id := record.get('event_location_external_id', None): pass
elif event_location_external_id := record.get('event_location_code', None): # elif event_location_external_id := record.get('event_location_code', None):
log.info('No event location external ID was found. Creating one with the location code...') # log.info('No event location external ID was found. Creating one with the location code...')
else: # else:
log.info('No event location external ID was found. Creating one with the name...') # log.info('No event location external ID was found. Creating one with the name...')
event_location_name = record.get('event_location_name', '') # event_location_name = record.get('event_location_name', '')
event_location_external_id = (str(event_id) + ':' + event_location_name).strip() # event_location_external_id = (str(event_id) + ':' + event_location_name).strip()
log.debug(f'Event Location External ID: {event_location_external_id}') # log.debug(f'Event Location External ID: {event_location_external_id}')
if event_session_external_id := record.get('event_session_external_id', None): pass # if event_session_external_id := record.get('event_session_external_id', None): pass
else: # else:
log.error('No event session external ID was found.') # log.error('No event session external ID was found.')
continue # continue
if event_presentation_external_id := record.get('event_presentation_external_id', None): pass # if event_presentation_external_id := record.get('event_presentation_external_id', None): pass
else: # else:
log.error('No event presentation external ID was found.') # log.error('No event presentation external ID was found.')
continue # continue
if event_presenter_external_id := record.get('event_presenter_external_id', None): pass # if event_presenter_external_id := record.get('event_presenter_external_id', None): pass
else: # else:
log.info('No event presenter external ID was found. Creating one...') # log.info('No event presenter external ID was found. Creating one...')
last_first_name = ', '.join( [record.get('event_presenter_family_name', ''), record.get('event_presenter_given_name', '')] ) # last_first_name = ', '.join( [record.get('event_presenter_family_name', ''), record.get('event_presenter_given_name', '')] )
event_presenter_external_id = (event_presentation_external_id + ':' + last_first_name).strip() # event_presenter_external_id = (event_presentation_external_id + ':' + last_first_name).strip()
log.debug(f'Event Presenter External ID: {event_presenter_external_id}') # log.debug(f'Event Presenter External ID: {event_presenter_external_id}')
data = {} # data = {}
data['account_id'] = account_id # data['account_id'] = account_id
data['event_id'] = event_id # data['event_id'] = event_id
# INSERT or UPDATE event_location # # INSERT or UPDATE event_location
data['event_location_external_id'] = event_location_external_id # data['event_location_external_id'] = event_location_external_id
sql = f""" # sql = f"""
SELECT * # SELECT *
FROM `v_event_location` AS `event_location` # FROM `v_event_location` AS `event_location`
WHERE /*event_location.account_id = :account_id # WHERE /*event_location.account_id = :account_id
AND*/ event_location.event_id = :event_id # AND*/ event_location.event_id = :event_id
AND event_location.external_id = :event_location_external_id # AND event_location.external_id = :event_location_external_id
LIMIT 1; # LIMIT 1;
""" # """
event_location_data = {} # event_location_data = {}
event_location_data['code'] = record.get('event_location_code', '') # event_location_data['code'] = record.get('event_location_code', '')
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case # event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_location_data['name'] = record.get('event_location_name', '') # event_location_data['name'] = record.get('event_location_name', '')
event_location_id = None # event_location_id = None
if event_location_rec_result := sql_select(data=data, sql=sql): # if event_location_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_location record # # Pull out IDs and UPDATE existing event_location record
event_location_rec = event_location_rec_result # event_location_rec = event_location_rec_result
event_location_id = event_location_rec.get('event_location_id', None) # event_location_id = event_location_rec.get('event_location_id', None)
event_location_id_random = event_location_rec.get('event_location_id_random', None) # event_location_id_random = event_location_rec.get('event_location_id_random', None)
event_location_name = event_location_rec.get('name', None) # event_location_name = event_location_rec.get('name', None)
log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}') # log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}')
event_location_data['id'] = event_location_id # event_location_data['id'] = event_location_id
log.debug(event_location_data) # log.debug(event_location_data)
if allow_updates: # if allow_updates:
if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'): # if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_up_result) # log.debug(event_location_obj_up_result)
else: # else:
log.warning(event_location_obj_up_result) # log.warning(event_location_obj_up_result)
pass # Probably nothing to update # pass # Probably nothing to update
else: # else:
log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}') # log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}')
event_location_data['external_id'] = event_location_external_id # NOTE: This should not change # event_location_data['external_id'] = event_location_external_id # NOTE: This should not change
log.debug(event_location_data) # log.debug(event_location_data)
if allow_inserts: # if allow_inserts:
if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'): # if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'):
log.debug(event_location_obj_in_result) # log.debug(event_location_obj_in_result)
event_location_id = event_location_obj_in_result # event_location_id = event_location_obj_in_result
else: # else:
log.warning(event_location_obj_in_result) # log.warning(event_location_obj_in_result)
continue # Something unexpected may have happened # continue # Something unexpected may have happened
# INSERT or UPDATE event_session # # INSERT or UPDATE event_session
data['event_session_external_id'] = event_session_external_id # data['event_session_external_id'] = event_session_external_id
sql = f""" # sql = f"""
SELECT * # SELECT *
FROM `v_event_session` AS `event_session` # FROM `v_event_session` AS `event_session`
WHERE event_session.account_id = :account_id # WHERE event_session.account_id = :account_id
AND event_session.event_id = :event_id # AND event_session.event_id = :event_id
AND event_session.external_id = :event_session_external_id # AND event_session.external_id = :event_session_external_id
LIMIT 1; # LIMIT 1;
""" # """
event_session_data = {} # event_session_data = {}
event_session_data['code'] = record.get('event_session_code', '') # event_session_data['code'] = record.get('event_session_code', '')
event_session_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case # event_session_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_session_data['event_location_id'] = event_location_id # INSERT and UPDATE the event_location_id in case the session was moved # event_session_data['event_location_id'] = event_location_id # INSERT and UPDATE the event_location_id in case the session was moved
event_session_data['name'] = record.get('event_session_name', '') # event_session_data['name'] = record.get('event_session_name', '')
event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') # event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '')
event_session_data['start_datetime'] = event_session_start_datetime # event_session_data['start_datetime'] = event_session_start_datetime
event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') # event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '')
event_session_data['end_datetime'] = event_session_end_datetime # event_session_data['end_datetime'] = event_session_end_datetime
if event_session_rec_result := sql_select(data=data, sql=sql): # if event_session_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_session record # # Pull out IDs and UPDATE existing event_session record
event_session_rec = event_session_rec_result # event_session_rec = event_session_rec_result
event_session_id = event_session_rec.get('event_session_id', None) # event_session_id = event_session_rec.get('event_session_id', None)
event_session_id_random = event_session_rec.get('event_session_id_random', None) # event_session_id_random = event_session_rec.get('event_session_id_random', None)
event_session_name = event_session_rec.get('name', None) # event_session_name = event_session_rec.get('name', None)
log.info(f'Found one record. Event Session ID: {event_session_id_random} Event Session Name: {event_session_name}') # log.info(f'Found one record. Event Session ID: {event_session_id_random} Event Session Name: {event_session_name}')
event_session_data['id'] = event_session_id # event_session_data['id'] = event_session_id
log.debug(event_session_data) # log.debug(event_session_data)
if allow_updates: # if allow_updates:
if event_session_obj_up_result := sql_update(data=event_session_data, table_name='event_session'): # if event_session_obj_up_result := sql_update(data=event_session_data, table_name='event_session'):
log.debug(event_session_obj_up_result) # log.debug(event_session_obj_up_result)
else: # else:
log.warning(event_session_obj_up_result) # log.warning(event_session_obj_up_result)
pass # Probably nothing to update # pass # Probably nothing to update
else: # else:
log.info(f'No record found that needs to be updated. Creating a new event session. Event Session External ID: {event_session_external_id}') # log.info(f'No record found that needs to be updated. Creating a new event session. Event Session External ID: {event_session_external_id}')
event_session_data['external_id'] = event_session_external_id # NOTE: This should not change # event_session_data['external_id'] = event_session_external_id # NOTE: This should not change
log.debug(event_session_data) # log.debug(event_session_data)
if allow_insert: # if allow_insert:
if event_session_obj_in_result := sql_insert(data=event_session_data, table_name='event_session'): # if event_session_obj_in_result := sql_insert(data=event_session_data, table_name='event_session'):
log.debug(event_session_obj_in_result) # log.debug(event_session_obj_in_result)
event_session_id = event_session_obj_in_result # event_session_id = event_session_obj_in_result
else: # else:
log.warning(event_session_obj_in_result) # log.warning(event_session_obj_in_result)
continue # Something unexpected may have happened # continue # Something unexpected may have happened
# INSERT or UPDATE event_presentation # # INSERT or UPDATE event_presentation
data['event_presentation_external_id'] = event_presentation_external_id # data['event_presentation_external_id'] = event_presentation_external_id
sql = f""" # sql = f"""
SELECT * # SELECT *
FROM `v_event_presentation` AS `event_presentation` # FROM `v_event_presentation` AS `event_presentation`
WHERE event_presentation.event_id = :event_id # WHERE event_presentation.event_id = :event_id
AND event_presentation.external_id = :event_presentation_external_id # AND event_presentation.external_id = :event_presentation_external_id
LIMIT 1; # LIMIT 1;
""" # """
event_presentation_data = {} # event_presentation_data = {}
event_presentation_data['code'] = record.get('event_presentation_code', '') # event_presentation_data['code'] = record.get('event_presentation_code', '')
event_presentation_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case # event_presentation_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_presentation_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved # event_presentation_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved
event_presentation_data['name'] = record.get('event_presentation_name', '') # event_presentation_data['name'] = record.get('event_presentation_name', '')
event_presentation_start_datetime = record.get('event_presentation_start_date', '') + ' ' + record.get('event_presentation_start_time', '') # event_presentation_start_datetime = record.get('event_presentation_start_date', '') + ' ' + record.get('event_presentation_start_time', '')
event_presentation_data['start_datetime'] = event_presentation_start_datetime # event_presentation_data['start_datetime'] = event_presentation_start_datetime
event_presentation_end_datetime = record.get('event_presentation_end_date', '') + ' ' + record.get('event_presentation_end_time', '') # event_presentation_end_datetime = record.get('event_presentation_end_date', '') + ' ' + record.get('event_presentation_end_time', '')
event_presentation_data['end_datetime'] = event_presentation_end_datetime # event_presentation_data['end_datetime'] = event_presentation_end_datetime
event_presentation_data['sort'] = record.get('event_presentation_sort', '') # event_presentation_data['sort'] = record.get('event_presentation_sort', '')
event_presentation_data['notes'] = record.get('event_presentation_notes', '') # event_presentation_data['notes'] = record.get('event_presentation_notes', '')
if event_presentation_rec_result := sql_select(data=data, sql=sql): # if event_presentation_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_presentation record # # Pull out IDs and UPDATE existing event_presentation record
event_presentation_rec = event_presentation_rec_result # event_presentation_rec = event_presentation_rec_result
event_presentation_id = event_presentation_rec.get('event_presentation_id', None) # event_presentation_id = event_presentation_rec.get('event_presentation_id', None)
event_presentation_id_random = event_presentation_rec.get('event_presentation_id_random', None) # event_presentation_id_random = event_presentation_rec.get('event_presentation_id_random', None)
event_presentation_name = event_presentation_rec.get('name', None) # event_presentation_name = event_presentation_rec.get('name', None)
log.info(f'Found one record. Event Presentation ID: {event_presentation_id_random} Event Presentation Name: {event_presentation_name}') # log.info(f'Found one record. Event Presentation ID: {event_presentation_id_random} Event Presentation Name: {event_presentation_name}')
event_presentation_data['id'] = event_presentation_id # event_presentation_data['id'] = event_presentation_id
log.debug(event_presentation_data) # log.debug(event_presentation_data)
if allow_updates: # if allow_updates:
if event_presentation_obj_up_result := sql_update(data=event_presentation_data, table_name='event_presentation'): # if event_presentation_obj_up_result := sql_update(data=event_presentation_data, table_name='event_presentation'):
log.debug(event_presentation_obj_up_result) # log.debug(event_presentation_obj_up_result)
else: # else:
log.warning(event_presentation_obj_up_result) # log.warning(event_presentation_obj_up_result)
pass # Probably nothing to update # pass # Probably nothing to update
else: # else:
log.info(f'No record found that needs to be updated. Creating a new event presentation. Event Presentation External ID: {event_presentation_external_id}') # log.info(f'No record found that needs to be updated. Creating a new event presentation. Event Presentation External ID: {event_presentation_external_id}')
event_presentation_data['external_id'] = event_presentation_external_id # NOTE: This should not change # event_presentation_data['external_id'] = event_presentation_external_id # NOTE: This should not change
log.debug(event_presentation_data) # log.debug(event_presentation_data)
if allow_inserts: # if allow_inserts:
if event_presentation_obj_in_result := sql_insert(data=event_presentation_data, table_name='event_presentation'): # if event_presentation_obj_in_result := sql_insert(data=event_presentation_data, table_name='event_presentation'):
log.debug(event_presentation_obj_in_result) # log.debug(event_presentation_obj_in_result)
event_presentation_id = event_presentation_obj_in_result # event_presentation_id = event_presentation_obj_in_result
else: # else:
log.warning(event_presentation_obj_in_result) # log.warning(event_presentation_obj_in_result)
continue # Something unexpected may have happened # continue # Something unexpected may have happened
# INSERT or UPDATE event_presenter # # INSERT or UPDATE event_presenter
data['event_presenter_external_id'] = event_presenter_external_id # data['event_presenter_external_id'] = event_presenter_external_id
sql = f""" # sql = f"""
SELECT * # SELECT *
FROM `v_event_presenter` AS `event_presenter` # FROM `v_event_presenter` AS `event_presenter`
WHERE event_presenter.event_id = :event_id # WHERE event_presenter.event_id = :event_id
AND event_presenter.external_id = :event_presenter_external_id # AND event_presenter.external_id = :event_presenter_external_id
LIMIT 1; # LIMIT 1;
""" # """
event_presenter_data = {} # event_presenter_data = {}
event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case # event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved # event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved
event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved # event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved
event_presenter_data['given_name'] = record.get('event_presenter_given_name', '') # event_presenter_data['given_name'] = record.get('event_presenter_given_name', '')
event_presenter_data['family_name'] = record.get('event_presenter_family_name', '') # event_presenter_data['family_name'] = record.get('event_presenter_family_name', '')
event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() # event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip()
event_presenter_data['designations'] = record.get('event_presenter_designations', '') # event_presenter_data['designations'] = record.get('event_presenter_designations', '')
event_presenter_data['email'] = record.get('event_presenter_email', '') # event_presenter_data['email'] = record.get('event_presenter_email', '')
event_presenter_data['notes'] = record.get('event_presenter_notes', '') # event_presenter_data['notes'] = record.get('event_presenter_notes', '')
if event_presenter_rec_result := sql_select(data=data, sql=sql): # if event_presenter_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_presenter record # # Pull out IDs and UPDATE existing event_presenter record
event_presenter_rec = event_presenter_rec_result # event_presenter_rec = event_presenter_rec_result
event_presenter_id = event_presenter_rec.get('event_presenter_id', None) # event_presenter_id = event_presenter_rec.get('event_presenter_id', None)
event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None) # event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None)
event_presenter_given_name = event_presenter_rec.get('given_name', None) # event_presenter_given_name = event_presenter_rec.get('given_name', None)
log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') # log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}')
event_presenter_data['id'] = event_presenter_id # event_presenter_data['id'] = event_presenter_id
log.debug(event_presenter_data) # log.debug(event_presenter_data)
if allow_updates: # if allow_updates:
if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): # if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_up_result) # log.debug(event_presenter_obj_up_result)
else: # else:
log.warning(event_presenter_obj_up_result) # log.warning(event_presenter_obj_up_result)
pass # Probably nothing to update # pass # Probably nothing to update
else: # else:
log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') # log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}')
event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should not change # event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should not change
log.debug(event_presenter_data) # log.debug(event_presenter_data)
if allow_inserts: # if allow_inserts:
if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): # if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_in_result) # log.debug(event_presenter_obj_in_result)
event_presenter_id = event_presenter_obj_in_result # event_presenter_id = event_presenter_obj_in_result
else: # else:
log.warning(event_presenter_obj_in_result) # log.warning(event_presenter_obj_in_result)
continue # Something unexpected may have happened # continue # Something unexpected may have happened
event_session_data_li.append(data) # event_session_data_li.append(data)
log.debug(f'Record processed: {event_session_id} {event_presentation_id} {event_presenter_full_name}') # log.debug(f'Record processed: {event_session_id} {event_presentation_id} {event_presenter_full_name}')
# End of loop # # End of loop
return mk_resp(data=event_session_data_li, response=response) # return mk_resp(data=event_session_data_li, response=response)
# ### BEGIN ### Event Importing ### event_importing_program_data() ### # ### BEGIN ### Event Importing ### event_importing_program_data() ###
@@ -402,31 +403,35 @@ async def event_importing_program_data(
full_file_path, full_file_path,
na_filter=False, na_filter=False,
dtype={ dtype={
'external_id': str, 'External ID': str, # Location related fields:
'event_location_external_id': str, 'event_location_code': str, 'event_location_sort': int, 'event_location_external_id': str, 'event_location_code': str, 'event_location_sort': int,
'location_external_id': str, 'location_code': str, 'location_code_(id)': str, 'location_sort': int,
# Presentation related fields:
'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int,
'presentation_external_id': str, 'presentation_code': str, 'presentation_code_(id)': str, 'presentation_sort': int,
# Presenter related fields:
'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_number': int, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_number': int, 'event_presenter_designations': str, 'event_presenter_sort': int,
'presenter_external_id': str, 'presenter_code': str, 'presenter_code_(id)': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str,
'email': str, 'Email Address': str,
# Session related fields:
'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int,
'location_external_id': str, 'location_code': str, 'location_sort': int,
'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int,
'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str,
'session_external_id': str, 'session_code': str, 'session_sort': int, 'session_external_id': str, 'session_code': str, 'session_sort': int,
'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str # 'external_id': str, 'External ID': str,
# 'source_code': str, 'Source ID': str,
} }
) )
df.rename(columns={ df.rename(columns={
'Source ID': 'source_code',
'External ID': 'external_id',
'Title': 'presenter_title_names', 'Title': 'presenter_title_names',
'Title Names': 'presenter_title_names', 'Title Names': 'presenter_title_names',
'Prefix': 'presenter_title_names', 'Prefix': 'presenter_title_names',
'Informal_name': 'presenter_informal_name', 'Informal_name': 'presenter_informal_name',
'Nickname': 'presenter_informal_name', 'Nickname': 'presenter_informal_name',
'given_name': 'presenter_given_name', 'given_name': 'presenter_given_name',
'given_name_(first)': 'presenter_given_name',
'given_name (first)': 'presenter_given_name', 'given_name (first)': 'presenter_given_name',
'Given Name': 'presenter_given_name', 'Given Name': 'presenter_given_name',
'First Name': 'presenter_given_name', 'First Name': 'presenter_given_name',
@@ -435,6 +440,7 @@ async def event_importing_program_data(
'family_name (last)': 'presenter_family_name', 'family_name (last)': 'presenter_family_name',
'Family Name': 'presenter_family_name', 'Family Name': 'presenter_family_name',
'Last Name': 'presenter_family_name', 'Last Name': 'presenter_family_name',
'family_name_(last)': 'presenter_family_name',
'Designations': 'presenter_designations', 'Designations': 'presenter_designations',
'Suffix': 'presenter_designations', 'Suffix': 'presenter_designations',
@@ -454,16 +460,26 @@ async def event_importing_program_data(
'Session Location': 'location_code', 'Session Location': 'location_code',
'location_title': 'location_name', 'location_title': 'location_name',
'Location Name': 'location_name', 'Location Name': 'location_name',
'location_name_(room)': 'location_name',
'session_location_name_(room)': 'location_name',
# 'session_location': 'location_name', # 'session_location': 'location_name',
'session_name_(title)': 'session_name',
'session_title': 'session_name', 'session_title': 'session_name',
'Session Code': 'session_code', 'Session Code': 'session_code',
'session_code_(id)': 'session_code',
'Session Name': 'session_name', 'Session Name': 'session_name',
'session_type_code_(id)': 'session_type_code',
'Session Type': 'session_type', 'Session Type': 'session_type',
'Session Hide': 'hide_session', 'Session Hide': 'hide_session',
'presentation_code_(id)': 'presentation_code',
'abstract_code': 'presentation_abstract_code',
'presentation_abstract_code_(id)': 'presentation_abstract_code',
'presentation_title': 'presentation_name', 'presentation_title': 'presentation_name',
'presentation_name_(title)': 'presentation_name',
'Presentation Code': 'presentation_code', 'Presentation Code': 'presentation_code',
'presenter_code_(id)': 'presenter_code',
'Presentation Name': 'presentation_name', 'Presentation Name': 'presentation_name',
'Presenter Code': 'presenter_code', 'Presenter Code': 'presenter_code',
@@ -472,6 +488,9 @@ async def event_importing_program_data(
'Presenter Notes': 'presenter_notes', 'Presenter Notes': 'presenter_notes',
'Notes': 'presenter_notes', 'Notes': 'presenter_notes',
# 'Source ID': 'source_code',
# 'External ID': 'external_id',
}, },
inplace = True) inplace = True)
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -493,7 +512,7 @@ async def event_importing_program_data(
if event_location_external_id := record.get('location_external_id'): if event_location_external_id := record.get('location_external_id'):
log.info('Using location_external_id for event_location_external_id') log.info('Using location_external_id for event_location_external_id')
elif event_location_external_id := record.get('location_code'): elif event_location_external_id := record.get('location_code', '').strip():
log.info('Using location_code for event_location_external_id') log.info('Using location_code for event_location_external_id')
else: else:
log.warning('No location external ID or code was found. Creating one with the name...') log.warning('No location external ID or code was found. Creating one with the name...')
@@ -504,7 +523,7 @@ async def event_importing_program_data(
if event_session_external_id := record.get('session_external_id'): if event_session_external_id := record.get('session_external_id'):
log.info('Using session_external_id for event_session_external_id') log.info('Using session_external_id for event_session_external_id')
elif event_session_external_id := record.get('session_code'): elif event_session_external_id := record.get('session_code').strip():
log.info('Using session_code for event_session_external_id') log.info('Using session_code for event_session_external_id')
else: else:
log.warning('No session external ID or code was found. Creating one with the name...') log.warning('No session external ID or code was found. Creating one with the name...')
@@ -514,7 +533,7 @@ async def event_importing_program_data(
if event_presentation_external_id := record.get('presentation_external_id'): if event_presentation_external_id := record.get('presentation_external_id'):
log.info('Using presentation_external_id for event_presentation_external_id') log.info('Using presentation_external_id for event_presentation_external_id')
elif event_presentation_external_id := record.get('presentation_code'): elif event_presentation_external_id := record.get('presentation_code').strip():
log.info('Using presentation_code for event_presentation_external_id') log.info('Using presentation_code for event_presentation_external_id')
else: else:
log.warning('No presentation external ID or code was found. Creating one with the name...') log.warning('No presentation external ID or code was found. Creating one with the name...')
@@ -524,13 +543,13 @@ async def event_importing_program_data(
if event_presenter_external_id := record.get('presenter_external_id'): if event_presenter_external_id := record.get('presenter_external_id'):
log.info('Using presenter_external_id for event_presenter_external_id') log.info('Using presenter_external_id for event_presenter_external_id')
elif event_presenter_external_id := record.get('presenter_code'): elif event_presenter_external_id := record.get('presenter_code').strip():
log.info('Using presenter_code for event_presenter_external_id') log.info('Using presenter_code for event_presenter_external_id')
elif event_presenter_external_id := record.get('presenter_number'): elif event_presenter_external_id := record.get('presenter_number'):
log.info('Using presenter_number for event_presenter_external_id') log.info('Using presenter_number for event_presenter_external_id')
else: else:
log.warning('No presenter external ID or code was found. Creating one with their family and given names...') log.warning('No presenter external ID or code was found. Creating one with their family and given names...')
event_presenter_last_first_name = ', '.join( [record.get('presenter_family_name', ''), record.get('presenter_given_name', '')] ) event_presenter_last_first_name = ', '.join( [record.get('presenter_family_name', '').strip(), record.get('presenter_given_name', '').strip()] )
# event_presenter_name = record.get('presenter_name', '').strip() # event_presenter_name = record.get('presenter_name', '').strip()
event_presenter_external_id = f'{event_presentation_external_id}:{event_presenter_last_first_name}' # .strip() event_presenter_external_id = f'{event_presentation_external_id}:{event_presenter_last_first_name}' # .strip()
log.debug(f'Event Presenter External ID: {event_presenter_external_id}') log.debug(f'Event Presenter External ID: {event_presenter_external_id}')
@@ -556,7 +575,10 @@ async def event_importing_program_data(
""" """
event_location_data = {} event_location_data = {}
event_location_data['code'] = record.get('location_code') if record.get('location_code'):
event_location_data['code'] = record.get('location_code').strip()
elif record.get('location_name'):
event_location_data['code'] = record.get('location_name').strip()
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
if record.get('location_name'): if record.get('location_name'):
event_location_data['name'] = record.get('location_name').strip() event_location_data['name'] = record.get('location_name').strip()
@@ -622,18 +644,19 @@ async def event_importing_program_data(
else: else:
event_session_data['name'] = None event_session_data['name'] = None
if record.get('session_type'): if record.get('session_type_code'):
event_session_data['type_code'] = record.get('session_type') event_session_data['type_code'] = record.get('session_type_code').strip()
# else: elif record.get('session_type'):
# event_session_data['type_code'] = None event_session_data['type_code'] = record.get('session_type').strip()
event_session_data['description'] = record.get('session_description') if record.get('session_description'):
event_session_data['description'] = record.get('session_description', '').strip()
event_session_data['start_datetime'] = record.get('session_start_datetime') event_session_data['start_datetime'] = record.get('session_start_datetime').strip()
# event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') # event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '')
# event_session_data['start_datetime'] = event_session_start_datetime # event_session_data['start_datetime'] = event_session_start_datetime
event_session_data['end_datetime'] = record.get('session_end_datetime') event_session_data['end_datetime'] = record.get('session_end_datetime').strip()
# event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') # event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '')
# event_session_data['end_datetime'] = event_session_end_datetime # event_session_data['end_datetime'] = event_session_end_datetime
@@ -643,7 +666,9 @@ async def event_importing_program_data(
event_session_data['hide'] = True event_session_data['hide'] = True
event_session_data['enable'] = True event_session_data['enable'] = True
event_session_data['notes'] = record.get('session_notes')
if record.get('session_notes'):
event_session_data['notes'] = record.get('session_notes', '').strip()
event_session_id = None event_session_id = None
if event_session_rec_result := sql_select(data=data, sql=sql): if event_session_rec_result := sql_select(data=data, sql=sql):
@@ -703,31 +728,35 @@ async def event_importing_program_data(
event_presentation_data['name'] = record.get('presentation_name').strip() event_presentation_data['name'] = record.get('presentation_name').strip()
else: else:
event_presentation_data['name'] = None event_presentation_data['name'] = None
event_presentation_data['description'] = record.get('presentation_description')
if record.get('presentation_description'):
event_presentation_data['description'] = record.get('presentation_description', '').strip()
if record.get('presentation_start_datetime'): if record.get('presentation_start_datetime'):
event_presentation_data['start_datetime'] = record.get('presentation_start_datetime') event_presentation_data['start_datetime'] = record.get('presentation_start_datetime').strip()
data['presentation_start_datetime'] = event_presentation_data['start_datetime'] data['presentation_start_datetime'] = event_presentation_data['start_datetime']
else: else:
event_presentation_data['start_datetime'] = None event_presentation_data['start_datetime'] = None
data['presentation_start_datetime'] = None data['presentation_start_datetime'] = None
if record.get('presentation_end_datetime'): if record.get('presentation_end_datetime'):
event_presentation_data['end_datetime'] = record.get('presentation_end_datetime') event_presentation_data['end_datetime'] = record.get('presentation_end_datetime').strip()
data['presentation_end_datetime'] = event_presentation_data['end_datetime'] data['presentation_end_datetime'] = event_presentation_data['end_datetime']
else: else:
event_presentation_data['end_datetime'] = None event_presentation_data['end_datetime'] = None
data['presentation_end_datetime'] = None data['presentation_end_datetime'] = None
if record.get('abstract_code'): if record.get('presentation_abstract_code'):
event_presentation_data['abstract_code'] = record.get('abstract_code') event_presentation_data['abstract_code'] = record.get('presentation_abstract_code').strip()
else: else:
event_presentation_data['abstract_code'] = None event_presentation_data['abstract_code'] = None
event_presentation_data['sort'] = record.get('presentation_sort') event_presentation_data['sort'] = record.get('presentation_sort')
event_presentation_data['enable'] = True event_presentation_data['enable'] = True
event_presentation_data['notes'] = record.get('presentation_notes')
if record.get('presentation_notes'):
event_presentation_data['notes'] = record.get('presentation_notes', '').strip()
event_presentation_id = None event_presentation_id = None
if (record.get('presentation_external_id') or record.get('presentation_code')) and record.get('presentation_name'): if (record.get('presentation_external_id') or record.get('presentation_code')) and record.get('presentation_name'):
@@ -810,7 +839,7 @@ async def event_importing_program_data(
event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved
event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved
if event_presenter_code := record.get('presenter_code'): if event_presenter_code := record.get('presenter_code').strip():
event_presenter_data['code'] = event_presenter_code event_presenter_data['code'] = event_presenter_code
elif event_presenter_code := record.get('presenter_number'): elif event_presenter_code := record.get('presenter_number'):
event_presenter_data['code'] = event_presenter_code event_presenter_data['code'] = event_presenter_code
@@ -820,20 +849,20 @@ async def event_importing_program_data(
elif event_presenter_number := isinstance(record.get('presenter_code'), int): elif event_presenter_number := isinstance(record.get('presenter_code'), int):
event_presenter_data['sort'] = event_presenter_number event_presenter_data['sort'] = event_presenter_number
event_presenter_data['title_names'] = record.get('presenter_title_names', '') event_presenter_data['title_names'] = record.get('presenter_title_names', '').strip()
event_presenter_data['informal_name'] = record.get('presenter_informal_name', '') event_presenter_data['informal_name'] = record.get('presenter_informal_name', '').strip()
event_presenter_data['given_name'] = record.get('presenter_given_name', '') event_presenter_data['given_name'] = record.get('presenter_given_name', '').strip()
event_presenter_data['middle_name'] = record.get('presenter_middle_name', '') event_presenter_data['middle_name'] = record.get('presenter_middle_name', '').strip()
event_presenter_data['family_name'] = record.get('presenter_family_name', '') event_presenter_data['family_name'] = record.get('presenter_family_name', '').strip()
event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip()
event_presenter_data['designations'] = record.get('presenter_designations', '') event_presenter_data['designations'] = record.get('presenter_designations', '').strip()
event_presenter_data['professional_title'] = record.get('presenter_professional_title', '') event_presenter_data['professional_title'] = record.get('presenter_professional_title', '').strip()
event_presenter_data['affiliations'] = record.get('presenter_affiliations', '') event_presenter_data['affiliations'] = record.get('presenter_affiliations', '').strip()
event_presenter_data['email'] = record.get('presenter_email', '') event_presenter_data['email'] = record.get('presenter_email', '').strip()
event_presenter_data['enable'] = True event_presenter_data['enable'] = True
event_presenter_data['notes'] = record.get('presenter_notes', '') event_presenter_data['notes'] = record.get('presenter_notes', '').strip()
event_presenter_id = None event_presenter_id = None
if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'): if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'):