From 2ef883984d358c19552a4be7f8292b70bc9d9b6b Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 15 Aug 2024 09:39:15 -0400 Subject: [PATCH] Updates to the event program data import script. --- app/routers/event_importing.py | 635 +++++++++++++++++---------------- 1 file changed, 332 insertions(+), 303 deletions(-) diff --git a/app/routers/event_importing.py b/app/routers/event_importing.py index 3a08843..ca6c383 100644 --- a/app/routers/event_importing.py +++ b/app/routers/event_importing.py @@ -28,304 +28,305 @@ from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() +# No longer needed? 2024-08-15 # Based on the program import template the clients are given. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-19 -@router.get('/event/importing/create_update_w_external_id', response_model=Resp_Body_Base) -async def event_importing_create_update_w_external_id( - response: Response = Response, - x_account_id: str = Header(...), - ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL - log.debug(locals()) +# @router.get('/event/importing/create_update_w_external_id', response_model=Resp_Body_Base) +# async def event_importing_create_update_w_external_id( +# response: Response = Response, +# x_account_id: str = Header(...), +# ): +# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL +# log.debug(locals()) - allow_inserts = False - allow_updates = False +# allow_inserts = False +# allow_updates = False - account_id = 2 # CMSC 2 8Gfxbxr19Nw - event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw - full_file_path = 'admin/temp/import_event_program_external_id.csv' +# account_id = 2 # CMSC 2 8Gfxbxr19Nw +# event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw +# full_file_path = 'admin/temp/import_event_program_external_id.csv' - df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str}) - log.debug(df) +# df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str}) +# log.debug(df) - df_dict = df.to_dict(orient='records') - - event_session_data_li = [] - - for record in df_dict: - - if event_location_external_id := record.get('event_location_external_id', None): pass - elif event_location_external_id := record.get('event_location_code', None): - log.info('No event location external ID was found. Creating one with the location code...') - else: - log.info('No event location external ID was found. Creating one with the name...') +# df_dict = df.to_dict(orient='records') + +# event_session_data_li = [] + +# for record in df_dict: + +# if event_location_external_id := record.get('event_location_external_id', None): pass +# elif event_location_external_id := record.get('event_location_code', None): +# log.info('No event location external ID was found. Creating one with the location code...') +# else: +# log.info('No event location external ID was found. Creating one with the name...') - event_location_name = record.get('event_location_name', '') +# event_location_name = record.get('event_location_name', '') - event_location_external_id = (str(event_id) + ':' + event_location_name).strip() - log.debug(f'Event Location External ID: {event_location_external_id}') +# event_location_external_id = (str(event_id) + ':' + event_location_name).strip() +# log.debug(f'Event Location External ID: {event_location_external_id}') - if event_session_external_id := record.get('event_session_external_id', None): pass - else: - log.error('No event session external ID was found.') - continue +# if event_session_external_id := record.get('event_session_external_id', None): pass +# else: +# log.error('No event session external ID was found.') +# continue - if event_presentation_external_id := record.get('event_presentation_external_id', None): pass - else: - log.error('No event presentation external ID was found.') - continue +# if event_presentation_external_id := record.get('event_presentation_external_id', None): pass +# else: +# log.error('No event presentation external ID was found.') +# continue - if event_presenter_external_id := record.get('event_presenter_external_id', None): pass - else: - log.info('No event presenter external ID was found. Creating one...') +# if event_presenter_external_id := record.get('event_presenter_external_id', None): pass +# else: +# log.info('No event presenter external ID was found. Creating one...') - last_first_name = ', '.join( [record.get('event_presenter_family_name', ''), record.get('event_presenter_given_name', '')] ) +# last_first_name = ', '.join( [record.get('event_presenter_family_name', ''), record.get('event_presenter_given_name', '')] ) - event_presenter_external_id = (event_presentation_external_id + ':' + last_first_name).strip() - log.debug(f'Event Presenter External ID: {event_presenter_external_id}') +# event_presenter_external_id = (event_presentation_external_id + ':' + last_first_name).strip() +# log.debug(f'Event Presenter External ID: {event_presenter_external_id}') - data = {} - data['account_id'] = account_id - data['event_id'] = event_id +# data = {} +# data['account_id'] = account_id +# data['event_id'] = event_id - # INSERT or UPDATE event_location - data['event_location_external_id'] = event_location_external_id +# # INSERT or UPDATE event_location +# data['event_location_external_id'] = event_location_external_id - sql = f""" - SELECT * - FROM `v_event_location` AS `event_location` - WHERE /*event_location.account_id = :account_id - AND*/ event_location.event_id = :event_id - AND event_location.external_id = :event_location_external_id - LIMIT 1; - """ +# sql = f""" +# SELECT * +# FROM `v_event_location` AS `event_location` +# WHERE /*event_location.account_id = :account_id +# AND*/ event_location.event_id = :event_id +# AND event_location.external_id = :event_location_external_id +# LIMIT 1; +# """ - event_location_data = {} - event_location_data['code'] = record.get('event_location_code', '') +# event_location_data = {} +# event_location_data['code'] = record.get('event_location_code', '') - event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case +# event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case - event_location_data['name'] = record.get('event_location_name', '') - - event_location_id = None - if event_location_rec_result := sql_select(data=data, sql=sql): - # Pull out IDs and UPDATE existing event_location record - event_location_rec = event_location_rec_result - event_location_id = event_location_rec.get('event_location_id', None) - event_location_id_random = event_location_rec.get('event_location_id_random', None) - event_location_name = event_location_rec.get('name', None) - log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}') - - event_location_data['id'] = event_location_id - - log.debug(event_location_data) +# event_location_data['name'] = record.get('event_location_name', '') + +# event_location_id = None +# if event_location_rec_result := sql_select(data=data, sql=sql): +# # Pull out IDs and UPDATE existing event_location record +# event_location_rec = event_location_rec_result +# event_location_id = event_location_rec.get('event_location_id', None) +# event_location_id_random = event_location_rec.get('event_location_id_random', None) +# event_location_name = event_location_rec.get('name', None) +# log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}') + +# event_location_data['id'] = event_location_id + +# log.debug(event_location_data) - if allow_updates: - if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'): - log.debug(event_location_obj_up_result) - else: - log.warning(event_location_obj_up_result) - pass # Probably nothing to update - else: - log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}') +# if allow_updates: +# if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'): +# log.debug(event_location_obj_up_result) +# else: +# log.warning(event_location_obj_up_result) +# pass # Probably nothing to update +# else: +# log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}') - event_location_data['external_id'] = event_location_external_id # NOTE: This should not change +# event_location_data['external_id'] = event_location_external_id # NOTE: This should not change - log.debug(event_location_data) +# log.debug(event_location_data) - if allow_inserts: - if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'): - log.debug(event_location_obj_in_result) - event_location_id = event_location_obj_in_result - else: - log.warning(event_location_obj_in_result) - continue # Something unexpected may have happened +# if allow_inserts: +# if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'): +# log.debug(event_location_obj_in_result) +# event_location_id = event_location_obj_in_result +# else: +# log.warning(event_location_obj_in_result) +# continue # Something unexpected may have happened - # INSERT or UPDATE event_session - data['event_session_external_id'] = event_session_external_id +# # INSERT or UPDATE event_session +# data['event_session_external_id'] = event_session_external_id - sql = f""" - SELECT * - FROM `v_event_session` AS `event_session` - WHERE event_session.account_id = :account_id - AND event_session.event_id = :event_id - AND event_session.external_id = :event_session_external_id - LIMIT 1; - """ +# sql = f""" +# SELECT * +# FROM `v_event_session` AS `event_session` +# WHERE event_session.account_id = :account_id +# AND event_session.event_id = :event_id +# AND event_session.external_id = :event_session_external_id +# LIMIT 1; +# """ - event_session_data = {} - event_session_data['code'] = record.get('event_session_code', '') +# event_session_data = {} +# event_session_data['code'] = record.get('event_session_code', '') - event_session_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case - event_session_data['event_location_id'] = event_location_id # INSERT and UPDATE the event_location_id in case the session was moved +# event_session_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case +# event_session_data['event_location_id'] = event_location_id # INSERT and UPDATE the event_location_id in case the session was moved - event_session_data['name'] = record.get('event_session_name', '') - - event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') - event_session_data['start_datetime'] = event_session_start_datetime - - event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') - event_session_data['end_datetime'] = event_session_end_datetime - - if event_session_rec_result := sql_select(data=data, sql=sql): - # Pull out IDs and UPDATE existing event_session record - event_session_rec = event_session_rec_result - event_session_id = event_session_rec.get('event_session_id', None) - event_session_id_random = event_session_rec.get('event_session_id_random', None) - event_session_name = event_session_rec.get('name', None) - log.info(f'Found one record. Event Session ID: {event_session_id_random} Event Session Name: {event_session_name}') - - event_session_data['id'] = event_session_id - - log.debug(event_session_data) - - if allow_updates: - if event_session_obj_up_result := sql_update(data=event_session_data, table_name='event_session'): - log.debug(event_session_obj_up_result) - else: - log.warning(event_session_obj_up_result) - pass # Probably nothing to update - else: - log.info(f'No record found that needs to be updated. Creating a new event session. Event Session External ID: {event_session_external_id}') - - event_session_data['external_id'] = event_session_external_id # NOTE: This should not change - - log.debug(event_session_data) - - if allow_insert: - if event_session_obj_in_result := sql_insert(data=event_session_data, table_name='event_session'): - log.debug(event_session_obj_in_result) - event_session_id = event_session_obj_in_result - else: - log.warning(event_session_obj_in_result) - continue # Something unexpected may have happened +# event_session_data['name'] = record.get('event_session_name', '') + +# event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') +# event_session_data['start_datetime'] = event_session_start_datetime + +# event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') +# event_session_data['end_datetime'] = event_session_end_datetime + +# if event_session_rec_result := sql_select(data=data, sql=sql): +# # Pull out IDs and UPDATE existing event_session record +# event_session_rec = event_session_rec_result +# event_session_id = event_session_rec.get('event_session_id', None) +# event_session_id_random = event_session_rec.get('event_session_id_random', None) +# event_session_name = event_session_rec.get('name', None) +# log.info(f'Found one record. Event Session ID: {event_session_id_random} Event Session Name: {event_session_name}') + +# event_session_data['id'] = event_session_id + +# log.debug(event_session_data) + +# if allow_updates: +# if event_session_obj_up_result := sql_update(data=event_session_data, table_name='event_session'): +# log.debug(event_session_obj_up_result) +# else: +# log.warning(event_session_obj_up_result) +# pass # Probably nothing to update +# else: +# log.info(f'No record found that needs to be updated. Creating a new event session. Event Session External ID: {event_session_external_id}') + +# event_session_data['external_id'] = event_session_external_id # NOTE: This should not change + +# log.debug(event_session_data) + +# if allow_insert: +# if event_session_obj_in_result := sql_insert(data=event_session_data, table_name='event_session'): +# log.debug(event_session_obj_in_result) +# event_session_id = event_session_obj_in_result +# else: +# log.warning(event_session_obj_in_result) +# continue # Something unexpected may have happened - # INSERT or UPDATE event_presentation - data['event_presentation_external_id'] = event_presentation_external_id - - sql = f""" - SELECT * - FROM `v_event_presentation` AS `event_presentation` - WHERE event_presentation.event_id = :event_id - AND event_presentation.external_id = :event_presentation_external_id - LIMIT 1; - """ - - event_presentation_data = {} - event_presentation_data['code'] = record.get('event_presentation_code', '') - - event_presentation_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case - event_presentation_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved - - event_presentation_data['name'] = record.get('event_presentation_name', '') - - event_presentation_start_datetime = record.get('event_presentation_start_date', '') + ' ' + record.get('event_presentation_start_time', '') - event_presentation_data['start_datetime'] = event_presentation_start_datetime - - event_presentation_end_datetime = record.get('event_presentation_end_date', '') + ' ' + record.get('event_presentation_end_time', '') - event_presentation_data['end_datetime'] = event_presentation_end_datetime - - event_presentation_data['sort'] = record.get('event_presentation_sort', '') - - event_presentation_data['notes'] = record.get('event_presentation_notes', '') - - if event_presentation_rec_result := sql_select(data=data, sql=sql): - # Pull out IDs and UPDATE existing event_presentation record - event_presentation_rec = event_presentation_rec_result - event_presentation_id = event_presentation_rec.get('event_presentation_id', None) - event_presentation_id_random = event_presentation_rec.get('event_presentation_id_random', None) - event_presentation_name = event_presentation_rec.get('name', None) - log.info(f'Found one record. Event Presentation ID: {event_presentation_id_random} Event Presentation Name: {event_presentation_name}') - - event_presentation_data['id'] = event_presentation_id - - log.debug(event_presentation_data) - - if allow_updates: - if event_presentation_obj_up_result := sql_update(data=event_presentation_data, table_name='event_presentation'): - log.debug(event_presentation_obj_up_result) - else: - log.warning(event_presentation_obj_up_result) - pass # Probably nothing to update - else: - log.info(f'No record found that needs to be updated. Creating a new event presentation. Event Presentation External ID: {event_presentation_external_id}') - - event_presentation_data['external_id'] = event_presentation_external_id # NOTE: This should not change +# # INSERT or UPDATE event_presentation +# data['event_presentation_external_id'] = event_presentation_external_id + +# sql = f""" +# SELECT * +# FROM `v_event_presentation` AS `event_presentation` +# WHERE event_presentation.event_id = :event_id +# AND event_presentation.external_id = :event_presentation_external_id +# LIMIT 1; +# """ + +# event_presentation_data = {} +# event_presentation_data['code'] = record.get('event_presentation_code', '') + +# event_presentation_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case +# event_presentation_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved + +# event_presentation_data['name'] = record.get('event_presentation_name', '') + +# event_presentation_start_datetime = record.get('event_presentation_start_date', '') + ' ' + record.get('event_presentation_start_time', '') +# event_presentation_data['start_datetime'] = event_presentation_start_datetime + +# event_presentation_end_datetime = record.get('event_presentation_end_date', '') + ' ' + record.get('event_presentation_end_time', '') +# event_presentation_data['end_datetime'] = event_presentation_end_datetime + +# event_presentation_data['sort'] = record.get('event_presentation_sort', '') + +# event_presentation_data['notes'] = record.get('event_presentation_notes', '') + +# if event_presentation_rec_result := sql_select(data=data, sql=sql): +# # Pull out IDs and UPDATE existing event_presentation record +# event_presentation_rec = event_presentation_rec_result +# event_presentation_id = event_presentation_rec.get('event_presentation_id', None) +# event_presentation_id_random = event_presentation_rec.get('event_presentation_id_random', None) +# event_presentation_name = event_presentation_rec.get('name', None) +# log.info(f'Found one record. Event Presentation ID: {event_presentation_id_random} Event Presentation Name: {event_presentation_name}') + +# event_presentation_data['id'] = event_presentation_id + +# log.debug(event_presentation_data) + +# if allow_updates: +# if event_presentation_obj_up_result := sql_update(data=event_presentation_data, table_name='event_presentation'): +# log.debug(event_presentation_obj_up_result) +# else: +# log.warning(event_presentation_obj_up_result) +# pass # Probably nothing to update +# else: +# log.info(f'No record found that needs to be updated. Creating a new event presentation. Event Presentation External ID: {event_presentation_external_id}') + +# event_presentation_data['external_id'] = event_presentation_external_id # NOTE: This should not change - log.debug(event_presentation_data) - - if allow_inserts: - if event_presentation_obj_in_result := sql_insert(data=event_presentation_data, table_name='event_presentation'): - log.debug(event_presentation_obj_in_result) - event_presentation_id = event_presentation_obj_in_result - else: - log.warning(event_presentation_obj_in_result) - continue # Something unexpected may have happened - - # INSERT or UPDATE event_presenter - data['event_presenter_external_id'] = event_presenter_external_id - - sql = f""" - SELECT * - FROM `v_event_presenter` AS `event_presenter` - WHERE event_presenter.event_id = :event_id - AND event_presenter.external_id = :event_presenter_external_id - LIMIT 1; - """ - - event_presenter_data = {} - event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case - event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved - event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved - - event_presenter_data['given_name'] = record.get('event_presenter_given_name', '') - event_presenter_data['family_name'] = record.get('event_presenter_family_name', '') - event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() - event_presenter_data['designations'] = record.get('event_presenter_designations', '') - - event_presenter_data['email'] = record.get('event_presenter_email', '') - - event_presenter_data['notes'] = record.get('event_presenter_notes', '') +# log.debug(event_presentation_data) + +# if allow_inserts: +# if event_presentation_obj_in_result := sql_insert(data=event_presentation_data, table_name='event_presentation'): +# log.debug(event_presentation_obj_in_result) +# event_presentation_id = event_presentation_obj_in_result +# else: +# log.warning(event_presentation_obj_in_result) +# continue # Something unexpected may have happened + +# # INSERT or UPDATE event_presenter +# data['event_presenter_external_id'] = event_presenter_external_id + +# sql = f""" +# SELECT * +# FROM `v_event_presenter` AS `event_presenter` +# WHERE event_presenter.event_id = :event_id +# AND event_presenter.external_id = :event_presenter_external_id +# LIMIT 1; +# """ + +# event_presenter_data = {} +# event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case +# event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved +# event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved + +# event_presenter_data['given_name'] = record.get('event_presenter_given_name', '') +# event_presenter_data['family_name'] = record.get('event_presenter_family_name', '') +# event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() +# event_presenter_data['designations'] = record.get('event_presenter_designations', '') + +# event_presenter_data['email'] = record.get('event_presenter_email', '') + +# event_presenter_data['notes'] = record.get('event_presenter_notes', '') - if event_presenter_rec_result := sql_select(data=data, sql=sql): - # Pull out IDs and UPDATE existing event_presenter record - event_presenter_rec = event_presenter_rec_result - event_presenter_id = event_presenter_rec.get('event_presenter_id', None) - event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None) - event_presenter_given_name = event_presenter_rec.get('given_name', None) - log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') - - event_presenter_data['id'] = event_presenter_id - - log.debug(event_presenter_data) +# if event_presenter_rec_result := sql_select(data=data, sql=sql): +# # Pull out IDs and UPDATE existing event_presenter record +# event_presenter_rec = event_presenter_rec_result +# event_presenter_id = event_presenter_rec.get('event_presenter_id', None) +# event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None) +# event_presenter_given_name = event_presenter_rec.get('given_name', None) +# log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') + +# event_presenter_data['id'] = event_presenter_id + +# log.debug(event_presenter_data) - if allow_updates: - if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): - log.debug(event_presenter_obj_up_result) - else: - log.warning(event_presenter_obj_up_result) - pass # Probably nothing to update - else: - log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') +# if allow_updates: +# if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): +# log.debug(event_presenter_obj_up_result) +# else: +# log.warning(event_presenter_obj_up_result) +# pass # Probably nothing to update +# else: +# log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') - event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should not change +# event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should not change - log.debug(event_presenter_data) +# log.debug(event_presenter_data) - if allow_inserts: - if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): - log.debug(event_presenter_obj_in_result) - event_presenter_id = event_presenter_obj_in_result - else: - log.warning(event_presenter_obj_in_result) - continue # Something unexpected may have happened - - event_session_data_li.append(data) - log.debug(f'Record processed: {event_session_id} {event_presentation_id} {event_presenter_full_name}') - # End of loop - - return mk_resp(data=event_session_data_li, response=response) +# if allow_inserts: +# if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): +# log.debug(event_presenter_obj_in_result) +# event_presenter_id = event_presenter_obj_in_result +# else: +# log.warning(event_presenter_obj_in_result) +# continue # Something unexpected may have happened + +# event_session_data_li.append(data) +# log.debug(f'Record processed: {event_session_id} {event_presentation_id} {event_presenter_full_name}') +# # End of loop + +# return mk_resp(data=event_session_data_li, response=response) # ### BEGIN ### Event Importing ### event_importing_program_data() ### @@ -402,31 +403,35 @@ async def event_importing_program_data( full_file_path, na_filter=False, dtype={ - 'external_id': str, 'External ID': str, + # Location related fields: 'event_location_external_id': str, 'event_location_code': str, 'event_location_sort': int, + 'location_external_id': str, 'location_code': str, 'location_code_(id)': str, 'location_sort': int, + + # Presentation related fields: 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, + 'presentation_external_id': str, 'presentation_code': str, 'presentation_code_(id)': str, 'presentation_sort': int, + + # Presenter related fields: 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_number': int, 'event_presenter_designations': str, 'event_presenter_sort': int, + 'presenter_external_id': str, 'presenter_code': str, 'presenter_code_(id)': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str, + 'email': str, 'Email Address': str, + + # Session related fields: 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, - - 'location_external_id': str, 'location_code': str, 'location_sort': int, - 'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int, - - 'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str, 'session_external_id': str, 'session_code': str, 'session_sort': int, - 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str + # 'external_id': str, 'External ID': str, + # 'source_code': str, 'Source ID': str, } ) df.rename(columns={ - 'Source ID': 'source_code', - 'External ID': 'external_id', - 'Title': 'presenter_title_names', 'Title Names': 'presenter_title_names', 'Prefix': 'presenter_title_names', 'Informal_name': 'presenter_informal_name', 'Nickname': 'presenter_informal_name', 'given_name': 'presenter_given_name', + 'given_name_(first)': 'presenter_given_name', 'given_name (first)': 'presenter_given_name', 'Given Name': 'presenter_given_name', 'First Name': 'presenter_given_name', @@ -435,6 +440,7 @@ async def event_importing_program_data( 'family_name (last)': 'presenter_family_name', 'Family Name': 'presenter_family_name', 'Last Name': 'presenter_family_name', + 'family_name_(last)': 'presenter_family_name', 'Designations': 'presenter_designations', 'Suffix': 'presenter_designations', @@ -454,16 +460,26 @@ async def event_importing_program_data( 'Session Location': 'location_code', 'location_title': 'location_name', 'Location Name': 'location_name', + 'location_name_(room)': 'location_name', + 'session_location_name_(room)': 'location_name', # 'session_location': 'location_name', + 'session_name_(title)': 'session_name', 'session_title': 'session_name', 'Session Code': 'session_code', + 'session_code_(id)': 'session_code', 'Session Name': 'session_name', + 'session_type_code_(id)': 'session_type_code', 'Session Type': 'session_type', 'Session Hide': 'hide_session', + 'presentation_code_(id)': 'presentation_code', + 'abstract_code': 'presentation_abstract_code', + 'presentation_abstract_code_(id)': 'presentation_abstract_code', 'presentation_title': 'presentation_name', + 'presentation_name_(title)': 'presentation_name', 'Presentation Code': 'presentation_code', + 'presenter_code_(id)': 'presenter_code', 'Presentation Name': 'presentation_name', 'Presenter Code': 'presenter_code', @@ -472,6 +488,9 @@ async def event_importing_program_data( 'Presenter Notes': 'presenter_notes', 'Notes': 'presenter_notes', + + # 'Source ID': 'source_code', + # 'External ID': 'external_id', }, inplace = True) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL @@ -493,7 +512,7 @@ async def event_importing_program_data( if event_location_external_id := record.get('location_external_id'): log.info('Using location_external_id for event_location_external_id') - elif event_location_external_id := record.get('location_code'): + elif event_location_external_id := record.get('location_code', '').strip(): log.info('Using location_code for event_location_external_id') else: log.warning('No location external ID or code was found. Creating one with the name...') @@ -504,7 +523,7 @@ async def event_importing_program_data( if event_session_external_id := record.get('session_external_id'): log.info('Using session_external_id for event_session_external_id') - elif event_session_external_id := record.get('session_code'): + elif event_session_external_id := record.get('session_code').strip(): log.info('Using session_code for event_session_external_id') else: log.warning('No session external ID or code was found. Creating one with the name...') @@ -514,7 +533,7 @@ async def event_importing_program_data( if event_presentation_external_id := record.get('presentation_external_id'): log.info('Using presentation_external_id for event_presentation_external_id') - elif event_presentation_external_id := record.get('presentation_code'): + elif event_presentation_external_id := record.get('presentation_code').strip(): log.info('Using presentation_code for event_presentation_external_id') else: log.warning('No presentation external ID or code was found. Creating one with the name...') @@ -524,13 +543,13 @@ async def event_importing_program_data( if event_presenter_external_id := record.get('presenter_external_id'): log.info('Using presenter_external_id for event_presenter_external_id') - elif event_presenter_external_id := record.get('presenter_code'): + elif event_presenter_external_id := record.get('presenter_code').strip(): log.info('Using presenter_code for event_presenter_external_id') elif event_presenter_external_id := record.get('presenter_number'): log.info('Using presenter_number for event_presenter_external_id') else: log.warning('No presenter external ID or code was found. Creating one with their family and given names...') - event_presenter_last_first_name = ', '.join( [record.get('presenter_family_name', ''), record.get('presenter_given_name', '')] ) + event_presenter_last_first_name = ', '.join( [record.get('presenter_family_name', '').strip(), record.get('presenter_given_name', '').strip()] ) # event_presenter_name = record.get('presenter_name', '').strip() event_presenter_external_id = f'{event_presentation_external_id}:{event_presenter_last_first_name}' # .strip() log.debug(f'Event Presenter External ID: {event_presenter_external_id}') @@ -556,7 +575,10 @@ async def event_importing_program_data( """ event_location_data = {} - event_location_data['code'] = record.get('location_code') + if record.get('location_code'): + event_location_data['code'] = record.get('location_code').strip() + elif record.get('location_name'): + event_location_data['code'] = record.get('location_name').strip() event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case if record.get('location_name'): event_location_data['name'] = record.get('location_name').strip() @@ -622,18 +644,19 @@ async def event_importing_program_data( else: event_session_data['name'] = None - if record.get('session_type'): - event_session_data['type_code'] = record.get('session_type') - # else: - # event_session_data['type_code'] = None + if record.get('session_type_code'): + event_session_data['type_code'] = record.get('session_type_code').strip() + elif record.get('session_type'): + event_session_data['type_code'] = record.get('session_type').strip() - event_session_data['description'] = record.get('session_description') + if record.get('session_description'): + event_session_data['description'] = record.get('session_description', '').strip() - event_session_data['start_datetime'] = record.get('session_start_datetime') + event_session_data['start_datetime'] = record.get('session_start_datetime').strip() # event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') # event_session_data['start_datetime'] = event_session_start_datetime - event_session_data['end_datetime'] = record.get('session_end_datetime') + event_session_data['end_datetime'] = record.get('session_end_datetime').strip() # event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') # event_session_data['end_datetime'] = event_session_end_datetime @@ -643,7 +666,9 @@ async def event_importing_program_data( event_session_data['hide'] = True event_session_data['enable'] = True - event_session_data['notes'] = record.get('session_notes') + + if record.get('session_notes'): + event_session_data['notes'] = record.get('session_notes', '').strip() event_session_id = None if event_session_rec_result := sql_select(data=data, sql=sql): @@ -703,31 +728,35 @@ async def event_importing_program_data( event_presentation_data['name'] = record.get('presentation_name').strip() else: event_presentation_data['name'] = None - event_presentation_data['description'] = record.get('presentation_description') + + if record.get('presentation_description'): + event_presentation_data['description'] = record.get('presentation_description', '').strip() if record.get('presentation_start_datetime'): - event_presentation_data['start_datetime'] = record.get('presentation_start_datetime') + event_presentation_data['start_datetime'] = record.get('presentation_start_datetime').strip() data['presentation_start_datetime'] = event_presentation_data['start_datetime'] else: event_presentation_data['start_datetime'] = None data['presentation_start_datetime'] = None if record.get('presentation_end_datetime'): - event_presentation_data['end_datetime'] = record.get('presentation_end_datetime') + event_presentation_data['end_datetime'] = record.get('presentation_end_datetime').strip() data['presentation_end_datetime'] = event_presentation_data['end_datetime'] else: event_presentation_data['end_datetime'] = None data['presentation_end_datetime'] = None - if record.get('abstract_code'): - event_presentation_data['abstract_code'] = record.get('abstract_code') + if record.get('presentation_abstract_code'): + event_presentation_data['abstract_code'] = record.get('presentation_abstract_code').strip() else: event_presentation_data['abstract_code'] = None event_presentation_data['sort'] = record.get('presentation_sort') event_presentation_data['enable'] = True - event_presentation_data['notes'] = record.get('presentation_notes') + + if record.get('presentation_notes'): + event_presentation_data['notes'] = record.get('presentation_notes', '').strip() event_presentation_id = None if (record.get('presentation_external_id') or record.get('presentation_code')) and record.get('presentation_name'): @@ -810,7 +839,7 @@ async def event_importing_program_data( event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved - if event_presenter_code := record.get('presenter_code'): + if event_presenter_code := record.get('presenter_code').strip(): event_presenter_data['code'] = event_presenter_code elif event_presenter_code := record.get('presenter_number'): event_presenter_data['code'] = event_presenter_code @@ -820,20 +849,20 @@ async def event_importing_program_data( elif event_presenter_number := isinstance(record.get('presenter_code'), int): event_presenter_data['sort'] = event_presenter_number - event_presenter_data['title_names'] = record.get('presenter_title_names', '') - event_presenter_data['informal_name'] = record.get('presenter_informal_name', '') - event_presenter_data['given_name'] = record.get('presenter_given_name', '') - event_presenter_data['middle_name'] = record.get('presenter_middle_name', '') - event_presenter_data['family_name'] = record.get('presenter_family_name', '') + event_presenter_data['title_names'] = record.get('presenter_title_names', '').strip() + event_presenter_data['informal_name'] = record.get('presenter_informal_name', '').strip() + event_presenter_data['given_name'] = record.get('presenter_given_name', '').strip() + event_presenter_data['middle_name'] = record.get('presenter_middle_name', '').strip() + event_presenter_data['family_name'] = record.get('presenter_family_name', '').strip() event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() - event_presenter_data['designations'] = record.get('presenter_designations', '') - event_presenter_data['professional_title'] = record.get('presenter_professional_title', '') - event_presenter_data['affiliations'] = record.get('presenter_affiliations', '') + event_presenter_data['designations'] = record.get('presenter_designations', '').strip() + event_presenter_data['professional_title'] = record.get('presenter_professional_title', '').strip() + event_presenter_data['affiliations'] = record.get('presenter_affiliations', '').strip() - event_presenter_data['email'] = record.get('presenter_email', '') + event_presenter_data['email'] = record.get('presenter_email', '').strip() event_presenter_data['enable'] = True - event_presenter_data['notes'] = record.get('presenter_notes', '') + event_presenter_data['notes'] = record.get('presenter_notes', '').strip() event_presenter_id = None if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'):