diff --git a/app/db_sql.py b/app/db_sql.py index 1fc1b81..fbae860 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -108,6 +108,7 @@ def sql_connect( # ### BEGIN ### API DB SQL ### sql_insert() ### # NOTE: Possible future change... Try to look up a duplicate entry if there is one on INSERT. +# Returns the auto number ID of the record inserted, or returns None if there was likely a duplicate record, or False if there was a problem of some kind. # Updated 2023-02-08 @logger_reset def sql_insert( diff --git a/app/routers/event_importing.py b/app/routers/event_importing.py index 7ef96ee..2ddfc23 100644 --- a/app/routers/event_importing.py +++ b/app/routers/event_importing.py @@ -46,7 +46,7 @@ async def event_importing_create_update_w_external_id( event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw full_file_path = 'admin/temp/import_event_program_external_id.csv' - df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str}) + df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str}) log.debug(df) df_dict = df.to_dict(orient='records') @@ -414,11 +414,11 @@ async def event_importing_program_data( 'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str, 'session_external_id': str, 'session_code': str, 'session_sort': int, - 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str + 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str } ) df.rename(columns={ - 'Source ID': 'source_id', + 'Source ID': 'source_code', 'External ID': 'external_id', 'Title': 'presenter_title_names', @@ -762,90 +762,410 @@ async def event_importing_program_data( data['event_presentation_id'] = event_presentation_id # INSERT or UPDATE event_presenter - data['event_presenter_external_id'] = event_presenter_external_id - sql = f""" - SELECT * - FROM `v_event_presenter` AS `event_presenter` - WHERE event_presenter.event_presentation_id = :event_presentation_id - AND event_presenter.external_id = :event_presenter_external_id - LIMIT 1; - """ + # NEW - Way to handle a semicolon delimited list of presenters in a single cell + if presenter_full_name_li := record.get('presenter_full_name_li', None): + log.info(f'Found {presenter_full_name_li}') + presenter_full_name_li = presenter_full_name_li.split(';') + event_presenter_li = [] + for presenter_full_name in presenter_full_name_li: + event_presenter_record_results = process_event_presenter_w_full_name( + event_presentation_id = event_presentation_id, presenter_full_name=presenter_full_name + ) + presentation_summary_data = event_presenter_record_results + event_presenter_li.append(presentation_summary_data) - event_presenter_data = {} - event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case - event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved - event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved + data['event_presenter_li'] = event_presenter_li - if event_presenter_code := record.get('presenter_code'): - event_presenter_data['code'] = event_presenter_code - elif event_presenter_code := record.get('presenter_number'): - event_presenter_data['code'] = event_presenter_code + else: + # OLD - Way to handle a single presenter per row + data['event_presenter_external_id'] = event_presenter_external_id - if event_presenter_number := record.get('presenter_number'): - event_presenter_data['sort'] = event_presenter_number - elif event_presenter_number := isinstance(record.get('presenter_code'), int): - event_presenter_data['sort'] = event_presenter_number + sql = f""" + SELECT * + FROM `v_event_presenter` AS `event_presenter` + WHERE event_presenter.event_presentation_id = :event_presentation_id + AND event_presenter.external_id = :event_presenter_external_id + LIMIT 1; + """ - event_presenter_data['title_names'] = record.get('presenter_title_names', '') - event_presenter_data['informal_name'] = record.get('presenter_informal_name', '') - event_presenter_data['given_name'] = record.get('presenter_given_name', '') - event_presenter_data['middle_name'] = record.get('presenter_middle_name', '') - event_presenter_data['family_name'] = record.get('presenter_family_name', '') - event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() - event_presenter_data['designations'] = record.get('presenter_designations', '') - event_presenter_data['professional_title'] = record.get('presenter_professional_title', '') - event_presenter_data['affiliations'] = record.get('presenter_affiliations', '') + event_presenter_data = {} + event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case + event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved + event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved - event_presenter_data['email'] = record.get('presenter_email', '') + if event_presenter_code := record.get('presenter_code'): + event_presenter_data['code'] = event_presenter_code + elif event_presenter_code := record.get('presenter_number'): + event_presenter_data['code'] = event_presenter_code - event_presenter_data['enable'] = True - event_presenter_data['notes'] = record.get('presenter_notes', '') + if event_presenter_number := record.get('presenter_number'): + event_presenter_data['sort'] = event_presenter_number + elif event_presenter_number := isinstance(record.get('presenter_code'), int): + event_presenter_data['sort'] = event_presenter_number - event_presenter_id = None - if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'): - if event_presenter_rec_result := sql_select(data=data, sql=sql): - # Pull out IDs and UPDATE existing event_presenter record - event_presenter_rec = event_presenter_rec_result - event_presenter_id = event_presenter_rec.get('event_presenter_id') - event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random') - event_presenter_given_name = event_presenter_rec.get('given_name') - log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') + event_presenter_data['title_names'] = record.get('presenter_title_names', '') + event_presenter_data['informal_name'] = record.get('presenter_informal_name', '') + event_presenter_data['given_name'] = record.get('presenter_given_name', '') + event_presenter_data['middle_name'] = record.get('presenter_middle_name', '') + event_presenter_data['family_name'] = record.get('presenter_family_name', '') + event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() + event_presenter_data['designations'] = record.get('presenter_designations', '') + event_presenter_data['professional_title'] = record.get('presenter_professional_title', '') + event_presenter_data['affiliations'] = record.get('presenter_affiliations', '') - event_presenter_data['id'] = event_presenter_id + event_presenter_data['email'] = record.get('presenter_email', '') - log.debug(event_presenter_data) + event_presenter_data['enable'] = True + event_presenter_data['notes'] = record.get('presenter_notes', '') - if allow_updates: - if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): - log.debug(event_presenter_obj_up_result) - else: - log.warning(event_presenter_obj_up_result) - pass # Probably nothing to update - else: - log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') + event_presenter_id = None + if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'): + if event_presenter_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing event_presenter record + event_presenter_rec = event_presenter_rec_result + event_presenter_id = event_presenter_rec.get('event_presenter_id') + event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random') + event_presenter_given_name = event_presenter_rec.get('given_name') + log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') - event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should never change + event_presenter_data['id'] = event_presenter_id - log.debug(event_presenter_data) + log.debug(event_presenter_data) - if allow_inserts: - if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): - log.debug(event_presenter_obj_in_result) - event_presenter_id = event_presenter_obj_in_result - else: - log.warning(event_presenter_obj_in_result) - continue # Something unexpected may have happened + if allow_updates: + if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): + log.debug(event_presenter_obj_up_result) + else: + log.warning(event_presenter_obj_up_result) + pass # Probably nothing to update + else: + log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') - data['event_presenter_id'] = event_presenter_id - data['event_presenter_code'] = event_presenter_code - data['event_presenter_number'] = event_presenter_number - data['event_presenter_full_name'] = event_presenter_full_name + event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should never change - # End of the loop. Append to program list data results. + log.debug(event_presenter_data) - event_program_data_li.append(data) - log.info(f'Record processed: Session: {event_session_id} Presentation: {event_presentation_id} Presenter: {event_presenter_full_name}') + if allow_inserts: + if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): + log.debug(event_presenter_obj_in_result) + event_presenter_id = event_presenter_obj_in_result + else: + log.warning(event_presenter_obj_in_result) + continue # Something unexpected may have happened + + data['event_presenter_id'] = event_presenter_id + data['event_presenter_code'] = event_presenter_code + data['event_presenter_number'] = event_presenter_number + data['event_presenter_full_name'] = event_presenter_full_name + + # End of the loop. Append to program list data results. + + event_program_data_li.append(data) + log.info(f'Record processed: Session: {event_session_id} Presentation: {event_presentation_id} Presenter: {event_presenter_full_name}') return mk_resp(data=event_program_data_li, response=commons.response) # ### END ### Event Importing ### event_importing_program_data() ### + + +# ### BEGIN ### Event Importing ### pre_program_person_data_import() ### +# This is the new import endpoint for pre-program person data. Currently this is intended for use with LCI. +# Required fields: account_id (from commons header), event_id, file +# Updated 2024-06-09 +@router.post('/event/{event_id}/importing/pre_program_person_data', response_model=Resp_Body_Base) +async def pre_program_data_import( + event_id: str = Path(min_length=11, max_length=22), + file: UploadFile = File(...), + + begin_at: int = 0, + end_at: int = 20000, + + commons: Common_Route_Params = Depends(common_route_params), + ): + log.setLevel(logging.INFO) + + account_id = commons.x_account_id + + file_info = await save_file( + file = file, + account_id = account_id, + link_to_type = 'event', + link_to_id = event_id, + ) + if file_info['saved']: + log.info('File saved') + log.debug(file_info) + else: + log.error('Something may have gone wrong while saving the uploaded file?') + return mk_resp(data=None, status_code=500, response=commons.response) + + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + log.info(f'Hosted Files Path: {hosted_files_path}') + log.debug(shutil.disk_usage(hosted_files_path)) + + subdirectory_dest = os.path.join(hosted_files_path, file_info.get('subdirectory_path')) + log.debug(subdirectory_dest) + hash_filename = file_info.get('hash_sha256')+'.file' + full_file_path = pathlib.Path( os.path.join(subdirectory_dest, hash_filename) ) + log.debug(full_file_path) + + if full_file_path.exists(): + log.info(f'Full File Path: {full_file_path}') + else: + log.warning(f'Not found at full File Path: {full_file_path}') + return mk_resp(data=None, status_code=500, response=commons.response) + + df = pandas.read_csv( + full_file_path, + na_filter=False, + dtype={ + 'external_id': str, 'External ID': str, + 'source_code': str, 'Source ID': str, + 'email': str, 'Email Address': str, + 'given_name': str, 'Given Name': str, 'First Name': str, + 'given_name_[first]' : str, + 'middle_name': str, 'Middle Name': str, + 'family_name': str, 'Family Name': str, 'Last Name': str, + 'family_name_[last]': str, + 'title_names': str, 'Title Names': str, 'Prefix': str, + 'informal_name': str, 'Nickname': str, + 'professional_title': str, 'Professional Title': str, + 'affiliations': str, 'Affiliations': str, 'Company': str, + 'affiliation': str, 'Affiliation': str, + 'notes': str, 'Notes': str, + 'data_json': str, + } + ) + df.rename(columns={ + 'Source ID': 'source_code', + 'External ID': 'external_id', + 'Email': 'email', + 'Email Address': 'email', + 'Given Name': 'given_name', + 'given_name_[first]': 'given_name', + 'First Name': 'given_name', + 'Middle Name': 'middle_name', + 'Family Name': 'family_name', + 'Last Name': 'family_name', + 'family_name_[last]': 'family_name', + 'Title Names': 'title_names', + 'Prefix': 'title_names', + 'Nickname': 'informal_name', + 'Professional Title': 'professional_title', + 'Affiliations': 'affiliations', + 'Affiliation': 'affiliations', + 'affiliation': 'affiliations', + 'Company': 'affiliations', + 'Notes': 'notes', + }, + inplace = True) + log.debug(df) + + df_dict = df.to_dict(orient='records') + + loop_count = 0 + person_data_li = [] + + for record in df_dict: + log.info(f'Loop Count: {loop_count}') + loop_count = loop_count + 1 + if loop_count <= begin_at: continue + if loop_count > end_at: break + + log.debug(record) + + source_code = 'iMIS:CSV' + + person_record_results = process_person_data(account_id, source_code, record) + log.info(f'Record processed: {person_record_results}') + + person_data_li.append(person_record_results) + + return mk_resp(data=person_data_li, response=commons.response) +# ### END ### Event Importing ### pre_program_person_data_import() ### + + +# ### BEGIN ### Event Importing ### process_person_data() ### +# Updated 2024-06-09 +def process_person_data(account_id, source_code, record): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + data = {} + + data['account_id'] = account_id + data['source_code'] = source_code + + data['external_id'] = record.get('external_id', None) # joined data fields like name and email + data['external_sys_id'] = record.get('external_sys_id', None) # UUID or auto-incremented ID + if not data['external_id'] and data['external_sys_id']: + data['external_id'] = data['external_sys_id'] + if not data['external_sys_id'] and data['external_id']: + data['external_sys_id'] = data['external_id'] + + data['informal_name'] = record.get('informal_name', None) + data['title_names'] = record.get('title_names', None) + data['given_name'] = record.get('given_name', None) + data['middle_name'] = record.get('middle_name', None) + data['family_name'] = record.get('family_name', None) + data['designations'] = record.get('designations', None) + data['professional_title'] = record.get('professional_title', None) + data['affiliations'] = record.get('affiliations', None) + + data['primary_email'] = record.get('email', None) + + data['notes'] = record.get('notes', None) + + data['data_json'] = record.get('data_json', None) + + # Field names that are prefixed with json__ will be added to the data_json field as a key value pair. + json_kv = {} + for key in record: + if key.startswith('json__'): + json_kv[key[6:]] = record[key] + # data['data_json'] = json.dumps({key[6:]: record[key]}) + data['data_json'] = json.dumps(json_kv) + log.debug(data['data_json']) + + log.info(f'Person - External ID: {data["external_id"]} | Email: {data["primary_email"]}') + + # Check if the person record already exists to UPDATE, else INSERT + person_rec = None + person_id = None + person_id_random = None + if data['external_sys_id'] and data['external_id']: + if person_rec_result := sql_select(table_name='v_person', field_name='external_sys_id', field_value=data['external_sys_id']): + if not isinstance(person_rec_result, list): + person_rec = person_rec_result + person_id = person_rec.get('person_id', None) + person_id_random = person_rec.get('person_id_random', None) + else: + log.warning('Found more than one record') + log.warning(person_rec_result) + # Do nothing + return False # Something unexpected may have happened + elif person_rec_result := sql_select(table_name='v_person', field_name='external_id', field_value=data['external_id']): + if not isinstance(person_rec_result, list): + person_rec = person_rec_result + person_id = person_rec.get('person_id', None) + person_id_random = person_rec.get('person_id_random', None) + else: + log.warning('Found more than one record') + log.warning(person_rec_result) + # Do nothing + return False + else: + log.info('No record found that needs to be updated. Creating a new person.') + + if person_id: + data['id'] = person_id + log.debug(data) + + if person_obj_up_result := sql_update(data=data, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + return False + else: + if person_obj_in_result := sql_insert(data=data, table_name='person'): + log.debug(person_obj_in_result) + person_id = person_obj_in_result + else: + log.warning(person_obj_in_result) + return False + + data['id_random'] = person_id_random + log.debug(data) + + summary_data = { + 'id': person_id, + 'id_random': person_id_random, + 'external_id': data['external_id'], + 'external_sys_id': data['external_sys_id'], + 'given_name': data['given_name'], + 'primary_email': data['primary_email'], + } + + return summary_data +# ### END ### Event Importing ### process_person_data() ### + + +# This will try to look up the person record based on their full name and then update the event presenter record. If the person record is not found, it will return False. If the person record is found, it will return the person record data. +# Should we also pass event_id and event_session_id? +def process_event_presenter_w_full_name(event_presentation_id, presenter_full_name): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + # First - SQL SELECT to find the person record based on the full name + # If the person record is found, use the person_id to UPDATE the event_presenter record. In the future this will likely be a new table named person_profile. It will likely be a combination of the person, event_badge, and event_presenter tables. + if person_rec := sql_select(table_name='v_person', field_name='full_name', field_value=presenter_full_name): + if not isinstance(person_rec, list): + person_id = person_rec.get('person_id', None) + person_id_random = person_rec.get('person_id_random', None) + else: + log.warning('Found more than one record') + log.warning(person_rec) + return False + # If the person record is not found, return False + else: + log.warning('Person record not found') + return False + + # Second - SQL SELECT to find the event presenter record based on the event_presentation_id and the person_id returned from the first SQL SELECT + # If the event presenter record is found, UPDATE the record with the person_id + if event_presenter_rec := sql_select(table_name='v_event_presenter', field_name='event_presentation_id', field_value=event_presentation_id): + if not isinstance(event_presenter_rec, list): + event_presenter_id = event_presenter_rec.get('event_presenter_id', None) + event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None) + else: + log.warning('Found more than one record') + log.warning(event_presenter_rec) + return False + + data = {} + data['id'] = event_presenter_id + data['person_id'] = person_id + + if event_presenter_obj_up_result := sql_update(data=data, table_name='event_presenter'): + log.debug(event_presenter_obj_up_result) + else: + log.warning(event_presenter_obj_up_result) + return False + # If the event presenter record is not found, INSERT a new record with the person_id + else: + data = {} + data['event_presentation_id'] = event_presentation_id + data['person_id'] = person_id + + # This should fill out the event_presenter record based on the person record. This will likely be a new table named person_profile. It will likely be a combination of the person, event_badge, and event_presenter tables. + data['external_id'] = person_rec['external_id'] + # Do we know the presenter code (or number)? + # Do we know for_type or for_id? + # data['event_id'] = person_rec['event_id'] + # data['event_session_id'] = person_rec['event_session_id'] + # data['event_presentation_id'] = person_rec['event_presentation_id'] + + + data['informal_name'] = person_rec['informal_name'] + data['title_names'] = person_rec['title_names'] + data['given_name'] = person_rec['given_name'] + data['middle_name'] = person_rec['middle_name'] + data['family_name'] = person_rec['family_name'] + data['full_name'] = person_rec['full_name'] + + data['designations'] = person_rec['designations'] + data['professional_title'] = person_rec['professional_title'] + + data['affiliations'] = person_rec['affiliations'] + data['email'] = person_rec['primary_email'] + + + if event_presenter_obj_in_result := sql_insert(data=data, table_name='event_presenter'): + log.debug(event_presenter_obj_in_result) + event_presenter_id = event_presenter_obj_in_result + else: + log.warning(event_presenter_obj_in_result) + + + + return False \ No newline at end of file