import datetime, json, os, pathlib, pytz, secrets, shutil, time import pandas, xlrd # qrcode from fastapi import APIRouter, Body, Depends, File, Header, HTTPException, Query, Response, status, UploadFile from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string, common_route_params, Common_Route_Params from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3 # from app.methods.event_session_methods import create_update_event_session_obj_v4, get_event_session_rec_list, load_event_session_obj, update_event_session_obj # from app.methods.event_presentation_methods import create_update_event_presentation_obj_v4, get_event_presentation_rec_list, load_event_presentation_obj # from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4, get_event_presenter_rec_list, load_event_presenter_obj from app.methods.hosted_file_methods import load_hosted_file_obj, save_file from app.models.event_models import Event_Base # from app.models.event_location_models import Event_Location_Base from app.models.event_person_models import Event_Person_Base # from app.models.event_presentation_models import Event_Presentation_Base # from app.models.event_presenter_models import Event_Presenter_Base # from app.models.event_session_models import Event_Session_Base from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() # Based on the program import template the clients are given. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-19 # ### BEGIN ### Event Importing ### event_importing_program_data() ### # Based on the program import template the clients are given. # Create and update locations, sessions, presentations, and presenters as needed. # Updated 2023-01-12 @router.post('/event/{event_id}/badge/import', response_model=Resp_Body_Base) async def event_id_badge_import( event_id: str = Query(..., min_length=11, max_length=22), file: UploadFile = File(...), begin_at: int = 0, end_at: int = 20000, return_detail: bool = False, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_inserts = True allow_updates = True account_id = commons.x_account_id # event_location_id = None # event_session_id = None # event_presentation_id = None # event_presenter_id = None # Processing Config Options: # How should the external_id generation and matching be done? # external_sys_id, external_reg_id and given and family name # prefix external_id with related events (registration setups) from external system # external_id from: external_sys_id, external_reg_id event_id_random = event_id if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass else: return mk_resp(data=None, status_code=404, response=commons.response) link_to_type = 'event' link_to_id = event_id file_info = await save_file( file = file, account_id = account_id, # account_id_random = account_id_random, link_to_type = link_to_type, link_to_id = link_to_id, # link_to_id_random = link_to_id_random, # check_allowed_extension = check_allowed_extension, ) if file_info['saved']: log.info('File saved') log.debug(file_info) else: log.error('Something may have gone wrong while saving the uploaded file?') return mk_resp(data=None, status_code=500, response=commons.response) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL hosted_files_path = settings.FILES_PATH['hosted_files_root'] # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' log.info(f'Hosted Files Path: {hosted_files_path}') log.debug(shutil.disk_usage(hosted_files_path)) # full_file_path = 'admin/temp/import_event_program_external_id.csv' subdirectory_dest = os.path.join(hosted_files_path, file_info.get('subdirectory_path')) log.debug(subdirectory_dest) hash_filename = file_info.get('hash_sha256')+'.file' full_file_path = pathlib.Path( os.path.join(subdirectory_dest, hash_filename) ) # NOTE: Must use pathlib.Path to use .exists() log.debug(full_file_path) if full_file_path.exists(): log.info(f'Full File Path: {full_file_path}') else: log.warning(f'Not found at full File Path: {full_file_path}') return mk_resp(data=None, status_code=500, response=commons.response) # return mk_resp(data=file_info, response=commons.response) df = pandas.read_csv( full_file_path, na_filter=False, dtype={ 'external_id': str, 'External ID': str, # Must be unique per person; Generate if needed 'external_event_id': str, 'External Event ID': str, # Usually same for all registrants 'external_registration_id': str, 'External Registration ID': str, # Must be unique per registration (may be more than one person; usually guests) 'external_person_id': str, 'External Person ID': str, # Must be unique per person # 'external_sys_id': str, 'External Sys ID': str, # Must be unique per person 'event_badge_template_id': int, 'Event Badge Template ID': int, # An actual ID number # 'event_location_external_id': str, 'event_location_code': str, 'event_location_sort': int, # 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, # 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_number': int, 'event_presenter_designations': str, 'event_presenter_sort': int, # 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, # 'location_external_id': str, 'location_code': str, 'location_sort': int, # 'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int, # 'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, # 'session_external_id': str, 'session_code': str, 'session_sort': int, # 'source_id': str, 'Source ID': str, 'pronouns': str, 'Pronouns': str, # For badge rendering 'informal_name': str, 'Informal Name': str, # For badge rendering 'title_names': str, 'Title Names': str, # For badge rendering 'given_name': str, 'Given Name': str, # For badge rendering 'middle_name': str, 'Middle Name': str, # For badge rendering 'family_name': str, 'Family Name': str, # For badge rendering 'designations': str, 'Designations': str, # For badge rendering 'professional_title': str, 'Professional Title': str, # For badge rendering 'professional_title_override': str, 'display_professional_title': str, 'Display Professional Title': str, # For badge rendering override 'full_name': str, 'Full Name': str, 'full_name_override': str, 'display_name': str, 'Display Name': str, 'Full Name Override': str, # For badge rendering override 'affiliations': str, 'Affiliations': str, # For badge rendering override 'affiliations_override': str, 'display_affiliations': str, 'Display Affiliations': str, # For badge rendering override 'email': str, 'Email Address': str, 'phone': str, 'Phone': str, 'address_line_1': str, 'Address Line 1': str, 'address_line_2': str, 'Address Line 2': str, 'address_line_3': str, 'Address Line 3': str, 'city': str, 'City': str, 'country_subdivision_code': str, 'Country Subdivision Code': str, 'state_province': str, 'State Province': str, 'state_province_abb': str, 'State Province Abb': str, 'postal_code': str, 'Postal Code': str, 'country_alpha_2_code': str, 'Country Alpha 2 Code': str, 'country': str, 'Country': str, 'full_address': str, 'Full Address': str, 'location': str, 'Location': str, # For badge rendering 'location_override': str, 'display_location': str, 'Display Location': str, # For badge rendering override 'badge_type_code': str, 'Badge Type Code': str, # Must be mapped to a badge template ID 'badge_type_code_override': str, 'Badge Type Code Override': str, 'badge_type': str, 'Badge Type': str, 'badge_type_override': str, 'Badge Type Override': str, 'member_type_code': str, 'Member Type Code': str, 'member_type': str, 'Member Type': str, 'registration_type_code': str, 'Registration Type Code': str, 'registration_type': str, 'Registration Type': str, 'other_1': str, 'Other 1': str, 'other_2': str, 'Other 2': str, 'ticket_1_code': str, 'Ticket 1 Code': str, 'ticket_2_code': str, 'Ticket 2 Code': str, 'ticket_3_code': str, 'Ticket 3 Code': str, 'ticket_4_code': str, 'Ticket 4 Code': str, 'ticket_5_code': str, 'Ticket 5 Code': str, 'ticket_6_code': str, 'Ticket 6 Code': str, 'ticket_7_code': str, 'Ticket 7 Code': str, 'ticket_8_code': str, 'Ticket 8 Code': str, 'allow_tracking': str, 'Allow Tracking': str, 'agree_to_tc': str, 'Agree to TC': str, } ) df.rename(columns={ 'External ID': 'external_id', 'External Event ID': 'external_event_id', 'External Registration ID': 'external_registration_id', 'External Person ID': 'external_person_id', 'External Sys ID': 'external_person_id', 'Event Badge Template ID': 'event_badge_template_id', 'Pronouns': 'pronouns', 'Informal_name': 'informal_name', 'Title Names': 'title_names', 'Given Name': 'given_name', 'Middle Name': 'middle_name', 'Family Name': 'family_name', 'Designations': 'designations', 'Professional Title': 'professional_title', 'Full Name': 'full_name', 'Display Name': 'full_name_override', 'Full Name Override': 'full_name_override', 'Affiliations': 'affiliations', # 'Title': 'presenter_title_names', # 'Prefix': 'presenter_title_names', # 'Nickname': 'presenter_informal_name', # 'given_name (first)': 'presenter_given_name', # 'First Name': 'presenter_given_name', # 'family_name (last)': 'presenter_family_name', # 'Last Name': 'presenter_family_name', # 'Suffix': 'presenter_designations', 'Email': 'email', 'Email Address': 'email', 'CC Email Address': 'cc_email', 'Phone': 'phone', 'Location': 'location', 'Badge Type Code': 'badge_type_code', 'Badge Type': 'badge_type', 'Member Type Code': 'member_type_code', 'Member Type': 'member_type', 'Registration Type Code': 'registration_type_code', 'Registration Type': 'registration_type', 'Other 1': 'other_1', 'Other 2': 'other_2', 'Ticket 1 Code': 'ticket_1_code', 'Ticket 2 Code': 'ticket_2_code', 'Ticket 3 Code': 'ticket_3_code', 'Ticket 4 Code': 'ticket_4_code', 'Ticket 5 Code': 'ticket_5_code', 'Ticket 6 Code': 'ticket_6_code', 'Ticket 7 Code': 'ticket_7_code', 'Ticket 8 Code': 'ticket_8_code', 'Allow Tracking': 'allow_tracking', 'Agree to TC': 'agree_to_tc', # 'location_title': 'location_name', # 'Location Code': 'location_code', # 'Location Name': 'location_name', # 'session_location': 'location_name', # 'Session Location': 'location_name', # 'session_title': 'session_name', # 'Session Code': 'session_code', # 'Session Name': 'session_name', # 'presentation_title': 'presentation_name', # 'Presentation Code': 'presentation_code', # 'Presentation Name': 'presentation_name', # 'Presenter Code': 'code', # 'Presenter Number': 'number', # for sorting # 'Presenter Name': 'name', }, inplace = True) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(df) df_dict = df.to_dict(orient='records') log.info(f'Total record count: {len(df_dict)}') loop_count = 0 event_badge_person_li = [] event_badge_person_summary_li = [] log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL for record in df_dict: # ### Figure out the external IDs log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break if external_id := record.get('external_id'): log.info('Using given external_id for external_id') elif record.get('external_registration_id') and record.get('external_person_id'): log.info('Using external_registration_id and external_person_id for external_id') external_id = str(record.get('external_registration_id'))+':'+str(record.get('external_person_id')) elif record.get('external_event_id') and record.get('external_person_id'): log.info('Using external_event_id and external_person_id for external_id') external_id = str(record.get('external_event_id'))+':'+str(record.get('external_person_id')) elif record.get('external_person_id'): log.info('Using only external_person_id for external_id') external_id = str(record.get('external_person_id')) elif record.get('external_registration_id'): log.info('Using only external_registration_id for external_id') external_id = str(record.get('external_registration_id')) else: log.warning('No external ID was found or could safely be generated.') break log.debug(f'Event Badge External ID: {external_id}') event_person_summary = {} event_person_summary['event_id'] = event_id event_person_summary['event_id_random'] = event_id_random event_person_summary['external_id'] = external_id event_person_summary['given_name'] = record.get('given_name') event_person_summary['family_name'] = record.get('family_name') event_person_summary['email'] = record.get('email') event_person_data = {} event_person_data['account_id'] = account_id # Is this needed? event_person_data['event_id'] = event_id event_person_data['enable'] = True event_person_data['external_id'] = external_id event_person_data['external_event_id'] = record.get('external_event_id') event_person_data['external_registration_id'] = record.get('external_registration_id') # event_person_data['external_reg_id'] = record.get('external_registration_id') # Deprecated event_person_data['external_person_id'] = record.get('external_person_id') # event_person_data['external_sys_id'] = record.get('external_person_id') # Deprecated event_person_data['event_badge'] = {} event_person_data['event_person_profile'] = {} # event_person_data['event_registration'] = {} # Not currently used event_person_data['event_person_profile']['enable'] = True event_person_data['event_person_profile']['pronouns'] = record.get('pronouns') event_person_data['event_person_profile']['informal_name'] = record.get('informal_name') event_person_data['event_person_profile']['title_names'] = record.get('title_names') event_person_data['event_person_profile']['given_name'] = record.get('given_name') event_person_data['event_person_profile']['family_name'] = record.get('family_name') event_person_data['event_person_profile']['designations'] = record.get('designations') event_person_data['event_person_profile']['professional_title'] = record.get('professional_title') event_person_data['event_person_profile']['full_name'] = record.get('full_name') event_person_data['event_person_profile']['affiliations'] = record.get('affiliations') event_person_data['event_person_profile']['email'] = record.get('email') event_person_data['event_person_profile']['phone'] = record.get('phone') event_person_data['event_person_profile']['location'] = record.get('location') event_person_data['event_badge']['external_id'] = external_id event_person_data['event_badge']['external_event_id'] = record.get('external_event_id') event_person_data['event_badge']['external_registration_id'] = record.get('external_registration_id') # event_person_data['event_badge']['external_reg_id'] = record.get('external_registration_id') # Deprecated event_person_data['event_badge']['external_person_id'] = record.get('external_person_id') # event_person_data['event_badge']['external_sys_id'] = record.get('external_person_id') # Deprecated event_person_data['event_badge']['enable'] = True event_person_data['event_badge']['pronouns'] = record.get('pronouns') event_person_data['event_badge']['informal_name'] = record.get('informal_name') event_person_data['event_badge']['title_names'] = record.get('title_names') event_person_data['event_badge']['given_name'] = record.get('given_name') event_person_data['event_badge']['family_name'] = record.get('family_name') event_person_data['event_badge']['designations'] = record.get('designations') event_person_data['event_badge']['professional_title'] = record.get('professional_title') event_person_data['event_badge']['full_name'] = record.get('full_name') event_person_data['event_badge']['affiliations'] = record.get('affiliations') event_person_data['event_badge']['email'] = record.get('email') event_person_data['event_badge']['phone'] = record.get('phone') event_person_data['event_badge']['location'] = record.get('location') event_person_data['event_badge']['event_badge_template_id'] = 9 # record.get('event_badge_template_id') event_person_data['event_badge']['badge_type_code'] = record.get('badge_type_code') event_person_data['event_badge']['badge_type'] = record.get('badge_type') # email = None # city = None # country_subdivision_code = None # state_province = None # state_province_abb = None # country_alpha_2_code = None # country = None # city = record.get('city') # state_province = record.get('state_province') # country = record.get('country') # country_alpha_2_code = record.get('country_alpha_2_code') # if location := record.get('location'): # else: # location = f'{city} {state_province} {country_alpha_2_code}' event_badge_person_li.append(event_person_data) event_badge_person_summary_li.append(event_person_summary) # event_person_summary_li.append(event_person_summary_data) loop_count = loop_count + 1 sql_select_event_person = f""" SELECT id AS event_person_id, id_random AS event_person_id_random, external_id AS event_person_external_id, external_event_id AS event_person_external_event_id, external_registration_id AS event_person_external_registration_id, external_person_id AS event_person_external_person_id, event_badge_id AS event_badge_id, event_person_profile_id AS event_person_profile_id, event_registration_id AS event_registration_id FROM `event_person` AS `event_person` WHERE event_person.event_id = :event_id AND event_person.external_id = :external_id /*LIMIT 1*/; """ if event_person_result := sql_select(sql=sql_select_event_person, data=event_person_summary): if isinstance(event_person_result, list): log.error(f'Found more than one Event Person with the same External ID. Count: {len(event_person_result)}') # return False else: event_person_id = event_person_result.get('event_person_id') event_badge_id = event_person_result.get('event_badge_id') event_person_profile_id = event_person_result.get('event_person_profile_id') log.info(f'Found Event Person. Updating existing... Event Person ID: {event_person_id}') if create_event_person_obj_result := create_update_event_person_obj_v4( event_person_dict_obj = event_person_data, event_person_id = event_person_id, account_id = account_id, event_id = event_id, event_badge_id = event_badge_id, event_person_profile_id = event_person_profile_id, # create_sub_obj = create_sub_obj, # fail_any = fail_any, # return_outline = False, ): event_person_id = create_event_person_obj_result log.warning(f'Event Person updated. Event Person ID: {event_person_id}') else: log.warning(f'Event Person not updated. Event Person ID: {event_person_id}') log.debug(event_badge_obj_in_result) # return False else: log.info('No Event Person found. Creating new...') if create_event_person_obj_result := create_update_event_person_obj_v4( event_person_dict_obj = event_person_data, account_id = account_id, event_id = event_id, # create_sub_obj = create_sub_obj, # fail_any = fail_any, # return_outline = False, ): event_person_id = create_event_person_obj_result log.warning(f'Event Person created. Event Person ID: {event_person_id}') else: log.warning(f'Event Person not created.') log.debug(event_badge_obj_in_result) # return False if return_detail: return mk_resp(data=event_badge_person_li, status_message=f'Importing badges from file. Found {len(person_li)} badges.', response=commons.response) else: return mk_resp(data=event_badge_person_summary_li, status_message=f'Checked for badges from file. Found {len(event_badge_person_li)} badges.', response=commons.response)