import datetime, json, pytz, secrets, time import pandas, xlrd # qrcode from fastapi import APIRouter, Body, Depends, Header, HTTPException, Path, Query, Response, status from pydantic import BaseModel, EmailStr, Field, ValidationError from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string, common_route_params, Common_Route_Params from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.event_presentation_methods import create_update_event_presentation_obj_v4 from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4 from app.methods.event_session_methods import create_update_event_session_obj_v4 # from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3 from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail, get_event_file_save_local, get_event_slot_data from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash from app.models.event_presentation_models import Event_Presentation_Base from app.models.event_presenter_models import Event_Presenter_Base from app.models.event_session_models import Event_Session_Base from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() # ### BEGIN ### API Confex ### import_event_session_list() ### # processing time: 156 records @ 1457 seconds (sessions, locations, presentations, presenters, files) # Updated 2024-04-25 @router.get('/event/{e_confex_event_id}/import_reg', response_model=Resp_Body_Base) async def import_event_session_list( e_confex_event_id: str = Path(min_length=5, max_length=22), # For AAPOR: aapor_2023 (2024-05); aapor_2023 (2023-05) event_id: str = Path(min_length=11, max_length=22), # For AAPOR: MCz-Qm-48-j1 (1513 2024-05) 9jW-Db-SF-wt (1478 2023-05); x2H2P2MYlXU (1447 2022-05) # Account ID For AAPOR: j5EBhRDqPuw (20) begin_loop: int = 1, end_loop: int = 250, return_detail: bool = False, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) log.warning('Starting Confex event session list import...') account_id = commons.x_account_id account_id_random = commons.x_account_id_random if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass else: return mk_resp(data=None, status_code=404, status_message=f'The Event ID was not found. Event ID: {event_id}', response=commons.response) event_session_li = get_event_session_list() if event_session_li: log.warning(f'Total record count: {len(event_session_li)}') else: return mk_resp(data=None, status_code=404, status_message=f'Checked for event sessions in Confex. No Confex event sessions found.', response=commons.response) # return mk_resp(data=len(event_session_li), status_code=501, response=commons.response) # Not Implemented # return mk_resp(data=event_session_li, status_code=501, response=commons.response) # Not Implemented # BEGIN: Loop through the sessions pulled from Confex start_loop = begin_loop loop_count = 0 # log.debug(f'INITIAL LOOP COUNT {loop_count}') loop_limit = end_loop # 200 event_session_summary_li = [] for event_session in event_session_li: loop_count = loop_count + 1 # log.debug(f'START LOOP COUNT {loop_count}') # if loop_count < start_loop: continue if loop_count >= start_loop and loop_count <= loop_limit: log.warning(f'PASSING WITH LOOP COUNT {loop_count}') pass else: log.info(f'Fake BREAKING (continue) ON LOOP COUNT {loop_count}') continue # if start_loop < loop_count and loop_count > loop_limit: # log.warning(f'Fake BREAKING (continue) ON LOOP COUNT {loop_count}') # continue # elif loop_count > start_loop: # log.warning(f'PASSING WITH LOOP COUNT {loop_count}') # pass # else: # log.warning(f'CONTINUING (< start loop) ON LOOP COUNT {loop_count}') # continue log.warning(f'BEGIN: **** *** ** * Confex event session import loop #{loop_count} * ** *** ****') confex_session_id = event_session.get('id') # if confex_session_id in ('1181', '1110', '1100', '1076'): # '1110', '1100' # # log.setLevel(logging.DEBUG) # log.warning(f'**************** FOUND {confex_session_id} ******************') # else: # log.setLevel(logging.INFO) # log.warning(f'*** FOUND {confex_session_id} ***') # continue # *** Part 1: *** Deal with the event_location first # log.setLevel(logging.DEBUG) log.debug(event_session.get('ParentList_SlotData')) if event_session.get('ParentList_SlotData') and len(event_session.get('ParentList_SlotData')): confex_slot_id = event_session.get('ParentList_SlotData')[0] confex_slot_data = get_event_slot_data(confex_slot_id = confex_slot_id) log.debug(confex_slot_data) # event_location_external_id = confex_slot_id # This will not work because it is not unique per room. event_location_external_id = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option event_location_code = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option event_location_name = confex_slot_data.get('Room_Name') data = {} data['account_id'] = account_id data['event_id'] = event_id # ### INSERT or UPDATE event_location # NOTE: In the future it may make more sense to pull this location part out and or add a checked ID array so we can avoid SQL queries. data['event_location_external_id'] = event_location_external_id sql = f""" SELECT * FROM `v_event_location` AS `event_location` WHERE event_location.event_id = :event_id AND event_location.external_id = :event_location_external_id LIMIT 1; """ event_location_data = {} event_location_data['code'] = event_location_code event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case if event_location_name: event_location_data['name'] = event_location_name else: event_location_data['name'] = 'Unknown' event_location_data['enable'] = True event_location_id = None if event_location_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing event_location record event_location_rec = event_location_rec_result event_location_id = event_location_rec.get('event_location_id') event_location_id_random = event_location_rec.get('event_location_id_random') event_location_name = event_location_data.get('name') log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}') event_location_data['id'] = event_location_id log.debug(event_location_data) if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'): log.debug(event_location_obj_up_result) else: log.warning(event_location_obj_up_result) pass # Probably nothing to update else: log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}') event_location_data['external_id'] = event_location_external_id # NOTE: This should never change log.debug(event_location_data) if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'): log.debug(event_location_obj_in_result) event_location_id = event_location_obj_in_result else: log.warning(event_location_obj_in_result) continue # Something unexpected may have happened data['event_location_id'] = event_location_id event_session_summary = {} event_session_summary['event_session_external_id'] = event_session.get('id') event_session_summary['event_session_name'] = event_session.get('Title') session_time_parts = event_session.get('SortableTimeString').split('_') session_start_datetime_str = session_time_parts[0]+'T'+session_time_parts[1] session_end_datetime_str = session_time_parts[0]+'T'+session_time_parts[2] event_session_summary['event_session_start_datetime'] = session_start_datetime_str # event_session_summary['event_session_end_datetime'] = session_end_datetime_str event_session_summary['event_session_type'] = event_session.get('ProgramCode') event_session_summary['event_session_type_name'] = event_session.get('ProgramTitle') event_session_summary['event_session_subject_name'] = event_session.get('SubjectTitle') event_session_summary['event_session_file_list'] = event_session.get('Child_MediaFiles') event_session_summary['event_session_slot_data'] = event_session.get('ParentList_SlotData') event_session_summary['event_location_id'] = event_location_id event_session_data = {} event_session_data['external_id'] = event_session.get('id') event_session_data['name'] = event_session.get('Title') event_session_data['start_datetime'] = session_start_datetime_str event_session_data['end_datetime'] = session_end_datetime_str event_session_data['event_location_id'] = event_location_id # event_session_data['event_session_type'] = event_session.get('ProgramCode') # event_session_data['event_session_type_name'] = event_session.get('ProgramTitle') # event_session_data['event_session_subject_name'] = event_session.get('SubjectTitle') # event_session_data['event_session_file_list'] = event_session.get('Child_MediaFiles') # SortableTimeString - '2023-05-11_08:00:00_09:30:00' # ParentList_Day - ['Day/2023-05-11'] # Child_MediaFiles # ProgramCode # ProgramTitle # SubjectTitle # SubmitterFullName # TopDisplay - main author and affiliation? # TopDisplayWithPapers - all authors and affiliations confex_session_detail = get_event_session_detail(confex_session_id = confex_session_id) event_session_qry = {} event_session_qry['event_id'] = event_id event_session_qry['external_id'] = confex_session_id # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(event_session_qry) sql_select_event_session = f""" SELECT id AS event_session_id, id_random AS event_session_id_random, external_id AS event_session_external_id, event_location_id AS event_location_id FROM `event_session` AS `event_session` WHERE event_session.event_id = :event_id AND event_session.external_id = :external_id /*LIMIT 1*/; """ log.debug(sql_select_event_session) try: event_session_obj = Event_Session_Base(**event_session_data) log.debug(event_session_obj) except ValidationError as e: log.error(e.json()) return False if event_session_result := sql_select(sql=sql_select_event_session, data=event_session_qry): if isinstance(event_session_result, list): log.error(f'Found more than one Event Session with the same External ID. Count: {len(event_session_result)}') break # return False else: event_session_id = event_session_result.get('event_session_id') log.info(f'Found Event Session. Updating existing... Event Session ID: {event_session_id}') if create_event_session_obj_result := create_update_event_session_obj_v4( event_session_dict_obj = event_session_obj, event_session_id = event_session_id, event_id = event_id, ): event_session_id = create_event_session_obj_result log.warning(f'Event Session updated. Event Session ID: {event_session_id}') else: log.warning(f'Event Session not updated. Event Session ID: {event_session_id}') log.debug(create_event_session_obj_result) break # return False else: log.info('No Event Session found. Creating new...') if create_event_session_obj_result := create_update_event_session_obj_v4( event_session_dict_obj = event_session_obj, event_id = event_id, ): event_session_id = create_event_session_obj_result log.warning(f'Event Session created. Event Session ID: {event_session_id}') else: log.warning(f'Event Session not created.') log.debug(create_event_session_obj_result) break # return False event_session_data['event_session_id'] = event_session_id # _random event_session_data['event_presentation_list'] = None # Confex "Paper" if confex_session_detail: # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL # log.setLevel(logging.DEBUG) log.info(f'Got session detail for Confex session {confex_session_id}') log.debug(f'Session Detail: {confex_session_detail}') # Presentation List event_session_summary['event_presentation_list'] = [] event_session_data['event_presentation_list'] = [] for confex_presentation_paper in confex_session_detail.get('ChildList_Paper'): # Presentation Information confex_presentation_id = confex_presentation_paper # if confex_presentation_id in ('Paper/1928', 'Paper/1770', 'Paper/1214'): # '', '' # log.setLevel(logging.DEBUG) # log.warning(f'**************** FOUND {confex_presentation_id} ******************') # else: # log.setLevel(logging.INFO) # log.warning(f'*** FOUND {confex_presentation_id} ***') # continue event_presentation_data = {} event_presentation_data['external_id'] = confex_presentation_id log.info('Confex: Getting presentation detail...') confex_presentation_detail = get_event_presentation_detail(confex_session_id = confex_session_id, confex_presentation_id = confex_presentation_id) if confex_presentation_detail: # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(f'Presentation Detail: {confex_presentation_detail}') event_presentation_data['name'] = confex_presentation_detail.get('Title') presentation_time_parts = confex_presentation_detail.get('SortableTimeString').split('_') presentation_start_datetime_str = presentation_time_parts[0]+'T'+presentation_time_parts[1] presentation_end_datetime_str = presentation_time_parts[0]+'T'+presentation_time_parts[2] event_presentation_data['start_datetime'] = presentation_start_datetime_str event_presentation_data['end_datetime'] = presentation_end_datetime_str # event_presentation_data['event_presentation_start_end_datetime'] = confex_presentation_detail.get('SortableTimeString') event_presentation_qry = {} event_presentation_qry['event_session_id'] = event_session_id event_presentation_qry['external_id'] = confex_presentation_id # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(event_presentation_qry) sql_select_event_presentation = f""" SELECT id AS event_presentation_id, id_random AS event_presentation_id_random, external_id AS event_presentation_external_id FROM `event_presentation` AS `event_presentation` WHERE event_presentation.event_session_id = :event_session_id AND event_presentation.external_id = :external_id /*LIMIT 1*/; """ log.debug(sql_select_event_presentation) try: event_presentation_obj = Event_Presentation_Base(**event_presentation_data) log.warning(event_presentation_obj) except ValidationError as e: log.error(e.json()) return False if event_presentation_result := sql_select(sql=sql_select_event_presentation, data=event_presentation_qry): if isinstance(event_presentation_result, list): log.error(f'Found more than one Event Presentation with the same External ID. Count: {len(event_presentation_result)}') break # return False else: event_presentation_id = event_presentation_result.get('event_presentation_id') log.info(f'Found Event Presentation. Updating existing... Event Presentation ID: {event_presentation_id}') if create_event_presentation_obj_result := create_update_event_presentation_obj_v4( event_presentation_dict_obj = event_presentation_obj, event_presentation_id = event_presentation_id, event_id = event_id, event_session_id = event_session_id, ): event_presentation_id = create_event_presentation_obj_result log.warning(f'Event Presentation updated. Event Presentation ID: {event_presentation_id}') else: log.warning(f'Event Presentation not updated. Event Presentation ID: {event_presentation_id}') log.debug(create_event_presentation_obj_result) break # return False else: log.info('No Event Presentation found. Creating new...') if create_event_presentation_obj_result := create_update_event_presentation_obj_v4( event_presentation_dict_obj = event_presentation_obj, event_id = event_id, event_session_id = event_session_id, ): event_presentation_id = create_event_presentation_obj_result log.warning(f'Event Presentation created. Event Presentation ID: {event_presentation_id}') else: log.warning(f'Event Presentation not created.') log.debug(create_event_presentation_obj_result) break # return False event_presentation_data['event_presentation_id'] = event_presentation_id # _random # Presenter Information submitter_full_name = confex_presentation_detail.get('SubmitterFullName') log.info(f'Event Session/Presenter Submitter: {submitter_full_name}') event_presentation_data['event_session_submitter_full_name'] = submitter_full_name presenter_full_name = parse_name(confex_presentation_detail.get('TopDisplay')) log.info(f'Event Presenter Full Name: {presenter_full_name}') event_presentation_data['event_presenter_full_name'] = presenter_full_name # if presenter_full_name: # else: # event_presentation_data['event_presenter_full_name'] = None # Presenter List event_presentation_data['event_presenter_list'] = [] if len(confex_presentation_detail.get('ChildList_Person')): for confex_presenter_person in confex_presentation_detail.get('ChildList_Person'): confex_presenter_id = confex_presenter_person event_presenter_summary = {} event_presenter_summary['external_id'] = confex_presenter_id event_presenter_summary['full_name'] = confex_presentation_detail.get('SubmitterFullName') confex_presenter_detail = get_event_presenter_detail(confex_session_id = confex_session_id, confex_presentation_id = confex_presentation_id, confex_presenter_id = confex_presenter_id) # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL if confex_presenter_detail: log.debug(f'Presenter Detail: {confex_presenter_detail}') event_presenter_summary['given_name'] = confex_presenter_detail.get('FirstName') event_presenter_summary['middle_name'] = confex_presenter_detail.get('MiddleName') event_presenter_summary['family_name'] = confex_presenter_detail.get('LastName') # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL event_presentation_data['event_presenter_list'].append(event_presenter_summary) event_presenter_data_sql = {} if presenter_full_name: event_presenter_data_sql['full_name'] = presenter_full_name event_presenter_data_sql['given_name'] = presenter_full_name else: event_presenter_data_sql['full_name'] = submitter_full_name event_presenter_data_sql['given_name'] = submitter_full_name event_presenter_qry = {} event_presenter_qry['event_session_id'] = event_session_id event_presenter_qry['event_presentation_id'] = event_presentation_id if len(event_presentation_data['event_presenter_list']): event_presenter_qry['external_id'] = event_presentation_data['event_presenter_list'][0]['external_id'] event_presenter_data_sql['external_id'] = event_presentation_data['event_presenter_list'][0]['external_id'] event_presenter_data_sql['given_name'] = event_presentation_data['event_presenter_list'][0]['given_name'] event_presenter_data_sql['middle_name'] = event_presentation_data['event_presenter_list'][0]['middle_name'] event_presenter_data_sql['family_name'] = event_presentation_data['event_presenter_list'][0]['family_name'] else: event_presenter_qry['external_id'] = None # log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(event_presenter_qry) if event_presenter_qry['external_id']: sql_select_event_presenter = f""" SELECT id AS event_presenter_id, id_random AS event_presenter_id_random, external_id AS event_presenter_external_id FROM `event_presenter` AS `event_presenter` WHERE event_presenter.event_presentation_id = :event_presentation_id AND event_presenter.external_id = :external_id /*LIMIT 1*/; """ else: sql_select_event_presenter = f""" SELECT id AS event_presenter_id, id_random AS event_presenter_id_random, external_id AS event_presenter_external_id FROM `event_presenter` AS `event_presenter` WHERE event_presenter.event_presentation_id = :event_presentation_id AND event_presenter.external_id IS NULL /*LIMIT 1*/; """ log.debug(sql_select_event_presenter) try: event_presenter_obj = Event_Presenter_Base(**event_presenter_data_sql) except ValidationError as e: log.error(e.json()) return False log.debug(event_presenter_obj) event_presenter_id = None if event_presenter_result := sql_select(sql=sql_select_event_presenter, data=event_presenter_qry): if isinstance(event_presenter_result, list): log.error(f'Found more than one Event Presenter with the same External ID. Count: {len(event_presenter_result)}') break # return False else: event_presenter_id = event_presenter_result.get('event_presenter_id') log.info(f'Found Event Presenter. Updating existing... Event Presenter ID: {event_presenter_id}') if create_event_presenter_obj_result := create_update_event_presenter_obj_v4( event_presenter_dict_obj = event_presenter_obj, event_presenter_id = event_presenter_id, event_id = event_id, event_session_id = event_session_id, event_presentation_id = event_presentation_id, ): event_presenter_id = create_event_presenter_obj_result log.warning(f'Event Presenter updated. Event Presenter ID: {event_presenter_id}') else: log.warning(f'Event Presenter not updated. Event Presenter ID: {event_presenter_id}') log.debug(create_event_presenter_obj_result) break # return False else: log.info('No Event Presenter found. Creating new...') if create_event_presenter_obj_result := create_update_event_presenter_obj_v4( event_presenter_dict_obj = event_presenter_obj, event_id = event_id, event_session_id = event_session_id, event_presentation_id = event_presentation_id, ): event_presenter_id = create_event_presenter_obj_result log.warning(f'Event Presenter created. Event Presenter ID: {event_presenter_id}') else: log.warning(f'Event Presenter not created.') log.debug(create_event_presenter_obj_result) break # return False # Presentation File List for Presenter # NOTE: Only the files under ChildList_VendorFiles where used for AAPOR 2023. # NOTE: There are more files under ChildList_Files. This was ignored for AAPOR 2023. log.info('Get presentation files for a presenter...') log.debug(confex_presentation_detail.get('ChildList_VendorFiles')) event_presentation_data['confex_file_list'] = [] event_presentation_data['event_file_list'] = [] for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'): # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(confex_vendor_file) confex_file_id = confex_vendor_file log.info('Confex: Getting file detail...') confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id) event_file_data = {} file_info = {} # For locally saved file info below if confex_file_detail: # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(f'File Detail: {confex_file_detail}') event_file_data['filename'] = confex_file_detail.get('FileName') event_file_data['display_name'] = confex_file_detail.get('DisplayName') event_file_data['extension'] = confex_file_detail.get('Ext') event_file_data['size'] = confex_file_detail.get('size') # in bytes? event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation' event_file_data['url'] = confex_file_detail.get('URL') event_file_data['created_on'] = confex_file_detail.get('_originated') event_file_data['updated_on'] = confex_file_detail.get('_lastchanged') file_info = await get_event_file_save_local( url = confex_file_detail.get('URL'), confex_file_info = event_file_data, account_id = account_id, link_to_type = 'event_presenter', link_to_id = event_presenter_id, ) event_presentation_data['confex_file_list'].append(event_file_data) event_presentation_data['event_file_list'].append(file_info) # event_presenter_data['event_presenter_id'] = event_presenter_id # _random if len(event_presentation_data['event_presenter_list']): event_presentation_data['event_presenter_list'][0]['event_presenter_id'] = event_presenter_id # _random event_session_summary['event_presentation_list'].append(event_presentation_data) # event_session.get('Child_MediaFiles') else: # return mk_resp(data=None, status_code=404, status_message=f'Looked up session in Confex. No Confex event session found.', response=commons.response) log.warning(f'Looked up session {confex_session_id} in Confex. No Confex event session found.') # event_session_data['details'] = confex_session_detail event_session_summary_li.append(event_session_summary) # loop_count = loop_count + 1 # if confex_session_id == '1110': # log.setLevel(logging.DEBUG) # log.warning('**************** FOUND 1110 ******************') # break return mk_resp(data=event_session_summary_li, status_code=200, response=commons.response) # Partially implemented... def parse_name(string): start_tag = '' end_tag = '' start_index = string.find(start_tag) end_index = string.find(end_tag) if start_index != -1 and end_index != -1: return string[start_index + len(start_tag):end_index] else: return None