Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Getting ready for LCI importing of pre program data

Browse Source
This commit is contained in:
Scott Idem
2024-06-09 18:31:16 -04:00
parent a1579e62c5
commit 9140455795
2 changed files with 391 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/db_sql.py
View File

@@ -108,6 +108,7 @@ def sql_connect(
# ### BEGIN ### API DB SQL ### sql_insert() ###
# NOTE: Possible future change... Try to look up a duplicate entry if there is one on INSERT.
# Returns the auto number ID of the record inserted, or returns None if there was likely a duplicate record, or False if there was a problem of some kind.
# Updated 2023-02-08
@logger_reset
def sql_insert(

460
app/routers/event_importing.py
View File

@@ -46,7 +46,7 @@ async def event_importing_create_update_w_external_id(
event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw
full_file_path = 'admin/temp/import_event_program_external_id.csv'
df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str})
df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_designations': str, 'event_presenter_sort': int, 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, 'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str})
log.debug(df)
df_dict = df.to_dict(orient='records')
@@ -414,11 +414,11 @@ async def event_importing_program_data(
'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, 'presenter_given_name': str,
'session_external_id': str, 'session_code': str, 'session_sort': int,
'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str
'source_code': str, 'Source ID': str, 'email': str, 'Email Address': str
}
)
df.rename(columns={
'Source ID': 'source_id',
'Source ID': 'source_code',
'External ID': 'external_id',
'Title': 'presenter_title_names',
@@ -762,90 +762,410 @@ async def event_importing_program_data(
data['event_presentation_id'] = event_presentation_id
# INSERT or UPDATE event_presenter
data['event_presenter_external_id'] = event_presenter_external_id
sql = f"""
SELECT *
FROM `v_event_presenter` AS `event_presenter`
WHERE event_presenter.event_presentation_id = :event_presentation_id
AND event_presenter.external_id = :event_presenter_external_id
LIMIT 1;
"""
# NEW - Way to handle a semicolon delimited list of presenters in a single cell
if presenter_full_name_li := record.get('presenter_full_name_li', None):
log.info(f'Found {presenter_full_name_li}')
presenter_full_name_li = presenter_full_name_li.split(';')
event_presenter_li = []
for presenter_full_name in presenter_full_name_li:
event_presenter_record_results = process_event_presenter_w_full_name(
event_presentation_id = event_presentation_id, presenter_full_name=presenter_full_name
)
presentation_summary_data = event_presenter_record_results
event_presenter_li.append(presentation_summary_data)
event_presenter_data = {}
event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved
event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved
data['event_presenter_li'] = event_presenter_li
if event_presenter_code := record.get('presenter_code'):
event_presenter_data['code'] = event_presenter_code
elif event_presenter_code := record.get('presenter_number'):
event_presenter_data['code'] = event_presenter_code
else:
# OLD - Way to handle a single presenter per row
data['event_presenter_external_id'] = event_presenter_external_id
if event_presenter_number := record.get('presenter_number'):
event_presenter_data['sort'] = event_presenter_number
elif event_presenter_number := isinstance(record.get('presenter_code'), int):
event_presenter_data['sort'] = event_presenter_number
sql = f"""
SELECT *
FROM `v_event_presenter` AS `event_presenter`
WHERE event_presenter.event_presentation_id = :event_presentation_id
AND event_presenter.external_id = :event_presenter_external_id
LIMIT 1;
"""
event_presenter_data['title_names'] = record.get('presenter_title_names', '')
event_presenter_data['informal_name'] = record.get('presenter_informal_name', '')
event_presenter_data['given_name'] = record.get('presenter_given_name', '')
event_presenter_data['middle_name'] = record.get('presenter_middle_name', '')
event_presenter_data['family_name'] = record.get('presenter_family_name', '')
event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip()
event_presenter_data['designations'] = record.get('presenter_designations', '')
event_presenter_data['professional_title'] = record.get('presenter_professional_title', '')
event_presenter_data['affiliations'] = record.get('presenter_affiliations', '')
event_presenter_data = {}
event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved
event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved
event_presenter_data['email'] = record.get('presenter_email', '')
if event_presenter_code := record.get('presenter_code'):
event_presenter_data['code'] = event_presenter_code
elif event_presenter_code := record.get('presenter_number'):
event_presenter_data['code'] = event_presenter_code
event_presenter_data['enable'] = True
event_presenter_data['notes'] = record.get('presenter_notes', '')
if event_presenter_number := record.get('presenter_number'):
event_presenter_data['sort'] = event_presenter_number
elif event_presenter_number := isinstance(record.get('presenter_code'), int):
event_presenter_data['sort'] = event_presenter_number
event_presenter_id = None
if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'):
if event_presenter_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_presenter record
event_presenter_rec = event_presenter_rec_result
event_presenter_id = event_presenter_rec.get('event_presenter_id')
event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random')
event_presenter_given_name = event_presenter_rec.get('given_name')
log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}')
event_presenter_data['title_names'] = record.get('presenter_title_names', '')
event_presenter_data['informal_name'] = record.get('presenter_informal_name', '')
event_presenter_data['given_name'] = record.get('presenter_given_name', '')
event_presenter_data['middle_name'] = record.get('presenter_middle_name', '')
event_presenter_data['family_name'] = record.get('presenter_family_name', '')
event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip()
event_presenter_data['designations'] = record.get('presenter_designations', '')
event_presenter_data['professional_title'] = record.get('presenter_professional_title', '')
event_presenter_data['affiliations'] = record.get('presenter_affiliations', '')
event_presenter_data['id'] = event_presenter_id
event_presenter_data['email'] = record.get('presenter_email', '')
log.debug(event_presenter_data)
event_presenter_data['enable'] = True
event_presenter_data['notes'] = record.get('presenter_notes', '')
if allow_updates:
if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_up_result)
else:
log.warning(event_presenter_obj_up_result)
pass # Probably nothing to update
else:
log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}')
event_presenter_id = None
if (record.get('presenter_code') or record.get('presenter_number')) and record.get('presenter_given_name'):
if event_presenter_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing event_presenter record
event_presenter_rec = event_presenter_rec_result
event_presenter_id = event_presenter_rec.get('event_presenter_id')
event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random')
event_presenter_given_name = event_presenter_rec.get('given_name')
log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}')
event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should never change
event_presenter_data['id'] = event_presenter_id
log.debug(event_presenter_data)
log.debug(event_presenter_data)
if allow_inserts:
if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_in_result)
event_presenter_id = event_presenter_obj_in_result
else:
log.warning(event_presenter_obj_in_result)
continue # Something unexpected may have happened
if allow_updates:
if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_up_result)
else:
log.warning(event_presenter_obj_up_result)
pass # Probably nothing to update
else:
log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}')
data['event_presenter_id'] = event_presenter_id
data['event_presenter_code'] = event_presenter_code
data['event_presenter_number'] = event_presenter_number
data['event_presenter_full_name'] = event_presenter_full_name
event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should never change
# End of the loop. Append to program list data results.
log.debug(event_presenter_data)
event_program_data_li.append(data)
log.info(f'Record processed: Session: {event_session_id} Presentation: {event_presentation_id} Presenter: {event_presenter_full_name}')
if allow_inserts:
if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'):
log.debug(event_presenter_obj_in_result)
event_presenter_id = event_presenter_obj_in_result
else:
log.warning(event_presenter_obj_in_result)
continue # Something unexpected may have happened
data['event_presenter_id'] = event_presenter_id
data['event_presenter_code'] = event_presenter_code
data['event_presenter_number'] = event_presenter_number
data['event_presenter_full_name'] = event_presenter_full_name
# End of the loop. Append to program list data results.
event_program_data_li.append(data)
log.info(f'Record processed: Session: {event_session_id} Presentation: {event_presentation_id} Presenter: {event_presenter_full_name}')
return mk_resp(data=event_program_data_li, response=commons.response)
# ### END ### Event Importing ### event_importing_program_data() ###
# ### BEGIN ### Event Importing ### pre_program_person_data_import() ###
# This is the new import endpoint for pre-program person data. Currently this is intended for use with LCI.
# Required fields: account_id (from commons header), event_id, file
# Updated 2024-06-09
@router.post('/event/{event_id}/importing/pre_program_person_data', response_model=Resp_Body_Base)
async def pre_program_data_import(
event_id: str = Path(min_length=11, max_length=22),
file: UploadFile = File(...),
begin_at: int = 0,
end_at: int = 20000,
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.INFO)
account_id = commons.x_account_id
file_info = await save_file(
file = file,
account_id = account_id,
link_to_type = 'event',
link_to_id = event_id,
)
if file_info['saved']:
log.info('File saved')
log.debug(file_info)
else:
log.error('Something may have gone wrong while saving the uploaded file?')
return mk_resp(data=None, status_code=500, response=commons.response)
hosted_files_path = settings.FILES_PATH['hosted_files_root']
log.info(f'Hosted Files Path: {hosted_files_path}')
log.debug(shutil.disk_usage(hosted_files_path))
subdirectory_dest = os.path.join(hosted_files_path, file_info.get('subdirectory_path'))
log.debug(subdirectory_dest)
hash_filename = file_info.get('hash_sha256')+'.file'
full_file_path = pathlib.Path( os.path.join(subdirectory_dest, hash_filename) )
log.debug(full_file_path)
if full_file_path.exists():
log.info(f'Full File Path: {full_file_path}')
else:
log.warning(f'Not found at full File Path: {full_file_path}')
return mk_resp(data=None, status_code=500, response=commons.response)
df = pandas.read_csv(
full_file_path,
na_filter=False,
dtype={
'external_id': str, 'External ID': str,
'source_code': str, 'Source ID': str,
'email': str, 'Email Address': str,
'given_name': str, 'Given Name': str, 'First Name': str,
'given_name_[first]' : str,
'middle_name': str, 'Middle Name': str,
'family_name': str, 'Family Name': str, 'Last Name': str,
'family_name_[last]': str,
'title_names': str, 'Title Names': str, 'Prefix': str,
'informal_name': str, 'Nickname': str,
'professional_title': str, 'Professional Title': str,
'affiliations': str, 'Affiliations': str, 'Company': str,
'affiliation': str, 'Affiliation': str,
'notes': str, 'Notes': str,
'data_json': str,
}
)
df.rename(columns={
'Source ID': 'source_code',
'External ID': 'external_id',
'Email': 'email',
'Email Address': 'email',
'Given Name': 'given_name',
'given_name_[first]': 'given_name',
'First Name': 'given_name',
'Middle Name': 'middle_name',
'Family Name': 'family_name',
'Last Name': 'family_name',
'family_name_[last]': 'family_name',
'Title Names': 'title_names',
'Prefix': 'title_names',
'Nickname': 'informal_name',
'Professional Title': 'professional_title',
'Affiliations': 'affiliations',
'Affiliation': 'affiliations',
'affiliation': 'affiliations',
'Company': 'affiliations',
'Notes': 'notes',
},
inplace = True)
log.debug(df)
df_dict = df.to_dict(orient='records')
loop_count = 0
person_data_li = []
for record in df_dict:
log.info(f'Loop Count: {loop_count}')
loop_count = loop_count + 1
if loop_count <= begin_at: continue
if loop_count > end_at: break
log.debug(record)
source_code = 'iMIS:CSV'
person_record_results = process_person_data(account_id, source_code, record)
log.info(f'Record processed: {person_record_results}')
person_data_li.append(person_record_results)
return mk_resp(data=person_data_li, response=commons.response)
# ### END ### Event Importing ### pre_program_person_data_import() ###
# ### BEGIN ### Event Importing ### process_person_data() ###
# Updated 2024-06-09
def process_person_data(account_id, source_code, record):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
data = {}
data['account_id'] = account_id
data['source_code'] = source_code
data['external_id'] = record.get('external_id', None) # joined data fields like name and email
data['external_sys_id'] = record.get('external_sys_id', None) # UUID or auto-incremented ID
if not data['external_id'] and data['external_sys_id']:
data['external_id'] = data['external_sys_id']
if not data['external_sys_id'] and data['external_id']:
data['external_sys_id'] = data['external_id']
data['informal_name'] = record.get('informal_name', None)
data['title_names'] = record.get('title_names', None)
data['given_name'] = record.get('given_name', None)
data['middle_name'] = record.get('middle_name', None)
data['family_name'] = record.get('family_name', None)
data['designations'] = record.get('designations', None)
data['professional_title'] = record.get('professional_title', None)
data['affiliations'] = record.get('affiliations', None)
data['primary_email'] = record.get('email', None)
data['notes'] = record.get('notes', None)
data['data_json'] = record.get('data_json', None)
# Field names that are prefixed with json__ will be added to the data_json field as a key value pair.
json_kv = {}
for key in record:
if key.startswith('json__'):
json_kv[key[6:]] = record[key]
# data['data_json'] = json.dumps({key[6:]: record[key]})
data['data_json'] = json.dumps(json_kv)
log.debug(data['data_json'])
log.info(f'Person - External ID: {data["external_id"]} | Email: {data["primary_email"]}')
# Check if the person record already exists to UPDATE, else INSERT
person_rec = None
person_id = None
person_id_random = None
if data['external_sys_id'] and data['external_id']:
if person_rec_result := sql_select(table_name='v_person', field_name='external_sys_id', field_value=data['external_sys_id']):
if not isinstance(person_rec_result, list):
person_rec = person_rec_result
person_id = person_rec.get('person_id', None)
person_id_random = person_rec.get('person_id_random', None)
else:
log.warning('Found more than one record')
log.warning(person_rec_result)
# Do nothing
return False # Something unexpected may have happened
elif person_rec_result := sql_select(table_name='v_person', field_name='external_id', field_value=data['external_id']):
if not isinstance(person_rec_result, list):
person_rec = person_rec_result
person_id = person_rec.get('person_id', None)
person_id_random = person_rec.get('person_id_random', None)
else:
log.warning('Found more than one record')
log.warning(person_rec_result)
# Do nothing
return False
else:
log.info('No record found that needs to be updated. Creating a new person.')
if person_id:
data['id'] = person_id
log.debug(data)
if person_obj_up_result := sql_update(data=data, table_name='person'):
log.debug(person_obj_up_result)
else:
log.warning(person_obj_up_result)
return False
else:
if person_obj_in_result := sql_insert(data=data, table_name='person'):
log.debug(person_obj_in_result)
person_id = person_obj_in_result
else:
log.warning(person_obj_in_result)
return False
data['id_random'] = person_id_random
log.debug(data)
summary_data = {
'id': person_id,
'id_random': person_id_random,
'external_id': data['external_id'],
'external_sys_id': data['external_sys_id'],
'given_name': data['given_name'],
'primary_email': data['primary_email'],
}
return summary_data
# ### END ### Event Importing ### process_person_data() ###
# This will try to look up the person record based on their full name and then update the event presenter record. If the person record is not found, it will return False. If the person record is found, it will return the person record data.
# Should we also pass event_id and event_session_id?
def process_event_presenter_w_full_name(event_presentation_id, presenter_full_name):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# First - SQL SELECT to find the person record based on the full name
# If the person record is found, use the person_id to UPDATE the event_presenter record. In the future this will likely be a new table named person_profile. It will likely be a combination of the person, event_badge, and event_presenter tables.
if person_rec := sql_select(table_name='v_person', field_name='full_name', field_value=presenter_full_name):
if not isinstance(person_rec, list):
person_id = person_rec.get('person_id', None)
person_id_random = person_rec.get('person_id_random', None)
else:
log.warning('Found more than one record')
log.warning(person_rec)
return False
# If the person record is not found, return False
else:
log.warning('Person record not found')
return False
# Second - SQL SELECT to find the event presenter record based on the event_presentation_id and the person_id returned from the first SQL SELECT
# If the event presenter record is found, UPDATE the record with the person_id
if event_presenter_rec := sql_select(table_name='v_event_presenter', field_name='event_presentation_id', field_value=event_presentation_id):
if not isinstance(event_presenter_rec, list):
event_presenter_id = event_presenter_rec.get('event_presenter_id', None)
event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None)
else:
log.warning('Found more than one record')
log.warning(event_presenter_rec)
return False
data = {}
data['id'] = event_presenter_id
data['person_id'] = person_id
if event_presenter_obj_up_result := sql_update(data=data, table_name='event_presenter'):
log.debug(event_presenter_obj_up_result)
else:
log.warning(event_presenter_obj_up_result)
return False
# If the event presenter record is not found, INSERT a new record with the person_id
else:
data = {}
data['event_presentation_id'] = event_presentation_id
data['person_id'] = person_id
# This should fill out the event_presenter record based on the person record. This will likely be a new table named person_profile. It will likely be a combination of the person, event_badge, and event_presenter tables.
data['external_id'] = person_rec['external_id']
# Do we know the presenter code (or number)?
# Do we know for_type or for_id?
# data['event_id'] = person_rec['event_id']
# data['event_session_id'] = person_rec['event_session_id']
# data['event_presentation_id'] = person_rec['event_presentation_id']
data['informal_name'] = person_rec['informal_name']
data['title_names'] = person_rec['title_names']
data['given_name'] = person_rec['given_name']
data['middle_name'] = person_rec['middle_name']
data['family_name'] = person_rec['family_name']
data['full_name'] = person_rec['full_name']
data['designations'] = person_rec['designations']
data['professional_title'] = person_rec['professional_title']
data['affiliations'] = person_rec['affiliations']
data['email'] = person_rec['primary_email']
if event_presenter_obj_in_result := sql_insert(data=data, table_name='event_presenter'):
log.debug(event_presenter_obj_in_result)
event_presenter_id = event_presenter_obj_in_result
else:
log.warning(event_presenter_obj_in_result)
return False