import datetime, json, os, pathlib, pytz, secrets, shutil, time import pandas, xlrd # qrcode from fastapi import APIRouter, Body, Depends, File, Header, HTTPException, Path, Query, Response, status, UploadFile from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string, common_route_params, Common_Route_Params from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3 # from app.methods.event_session_methods import create_update_event_session_obj_v4, get_event_session_rec_list, load_event_session_obj, update_event_session_obj # from app.methods.event_presentation_methods import create_update_event_presentation_obj_v4, get_event_presentation_rec_list, load_event_presentation_obj # from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4, get_event_presenter_rec_list, load_event_presenter_obj from app.methods.hosted_file_methods import load_hosted_file_obj, save_file from app.models.event_models import Event_Base # from app.models.event_location_models import Event_Location_Base from app.models.event_person_models import Event_Person_Base # from app.models.event_presentation_models import Event_Presentation_Base # from app.models.event_presenter_models import Event_Presenter_Base # from app.models.event_session_models import Event_Session_Base from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() # Based on the program import template the clients are given. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-19 # ### BEGIN ### Event Importing ### event_importing_program_data() ### # Based on the program import template the clients are given. # Create and update locations, sessions, presentations, and presenters as needed. # Updated 2023-01-12 @router.post('/event/{event_id}/badge/import', response_model=Resp_Body_Base) async def event_id_badge_import( event_id: str = Path(min_length=11, max_length=22), file: UploadFile = File(...), begin_at: int = 0, end_at: int = 20000, return_detail: bool = False, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_inserts = True allow_updates = True account_id = commons.x_account_id # event_location_id = None # event_session_id = None # event_presentation_id = None # event_presenter_id = None # Processing Config Options: # How should the external_id generation and matching be done? # external_sys_id, external_reg_id and given and family name # prefix external_id with related events (registration setups) from external system # external_id from: external_sys_id, external_reg_id event_id_random = event_id if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass else: return mk_resp(data=None, status_code=404, response=commons.response) link_to_type = 'event' link_to_id = event_id file_info = await save_file( file = file, account_id = account_id, # account_id_random = account_id_random, link_to_type = link_to_type, link_to_id = link_to_id, # link_to_id_random = link_to_id_random, # check_allowed_extension = check_allowed_extension, ) if file_info['saved']: log.info('File saved') log.debug(file_info) else: log.error('Something may have gone wrong while saving the uploaded file?') return mk_resp(data=None, status_code=500, response=commons.response) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL hosted_files_path = settings.FILES_PATH['hosted_files_root'] # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' log.info(f'Hosted Files Path: {hosted_files_path}') log.debug(shutil.disk_usage(hosted_files_path)) # full_file_path = 'admin/temp/import_event_program_external_id.csv' subdirectory_dest = os.path.join(hosted_files_path, file_info.get('subdirectory_path')) log.debug(subdirectory_dest) hash_filename = file_info.get('hash_sha256')+'.file' full_file_path = pathlib.Path( os.path.join(subdirectory_dest, hash_filename) ) # NOTE: Must use pathlib.Path to use .exists() log.debug(full_file_path) if full_file_path.exists(): log.info(f'Full File Path: {full_file_path}') else: log.warning(f'Not found at full File Path: {full_file_path}') return mk_resp(data=None, status_code=500, response=commons.response) # return mk_resp(data=file_info, response=commons.response) df = pandas.read_csv( full_file_path, na_filter=False, dtype={ 'external_id': str, 'External ID': str, # Must be unique per person; Generate if needed 'external_event_id': str, 'External Event ID': str, # Usually same for all registrants 'external_registration_id': str, 'External Registration ID': str, # Must be unique per registration (may be more than one person; usually guests) 'external_person_id': str, 'External Person ID': str, # Must be unique per person # 'external_sys_id': str, 'External Sys ID': str, # Must be unique per person 'event_badge_template_id': int, 'Event Badge Template ID': int, # An actual ID number # 'event_location_external_id': str, 'event_location_code': str, 'event_location_sort': int, # 'event_presentation_external_id': str, 'event_presentation_code': str, 'event_presentation_sort': int, # 'event_presenter_external_id': str, 'event_presenter_code': str, 'event_presenter_number': int, 'event_presenter_designations': str, 'event_presenter_sort': int, # 'event_session_external_id': str, 'event_session_code': str, 'event_session_sort': int, # 'location_external_id': str, 'location_code': str, 'location_sort': int, # 'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int, # 'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int, # 'session_external_id': str, 'session_code': str, 'session_sort': int, # 'source_id': str, 'Source ID': str, 'pronouns': str, 'Pronouns': str, # For badge rendering 'informal_name': str, 'Informal Name': str, # For badge rendering 'title_names': str, 'Title Names': str, # For badge rendering 'given_name': str, 'Given Name': str, # For badge rendering 'middle_name': str, 'Middle Name': str, # For badge rendering 'family_name': str, 'Family Name': str, # For badge rendering 'designations': str, 'Designations': str, # For badge rendering 'professional_title': str, 'Professional Title': str, # For badge rendering 'professional_title_override': str, 'display_professional_title': str, 'Display Professional Title': str, # For badge rendering override 'full_name': str, 'Full Name': str, 'full_name_override': str, 'display_name': str, 'Display Name': str, 'Full Name Override': str, # For badge rendering override 'affiliations': str, 'Affiliations': str, # For badge rendering override 'affiliations_override': str, 'display_affiliations': str, 'Display Affiliations': str, # For badge rendering override 'email': str, 'Email Address': str, 'phone': str, 'Phone': str, 'address_line_1': str, 'Address Line 1': str, 'address_line_2': str, 'Address Line 2': str, 'address_line_3': str, 'Address Line 3': str, 'city': str, 'City': str, 'country_subdivision_code': str, 'Country Subdivision Code': str, 'state_province': str, 'State Province': str, 'state_province_abb': str, 'State Province Abb': str, 'postal_code': str, 'Postal Code': str, 'country_alpha_2_code': str, 'Country Alpha 2 Code': str, 'country': str, 'Country': str, 'full_address': str, 'Full Address': str, 'location': str, 'Location': str, # For badge rendering 'location_override': str, 'display_location': str, 'Display Location': str, # For badge rendering override 'badge_type_code': str, 'Badge Type Code': str, # Must be mapped to a badge template ID 'badge_type_code_override': str, 'Badge Type Code Override': str, 'badge_type': str, 'Badge Type': str, 'badge_type_override': str, 'Badge Type Override': str, 'member_type_code': str, 'Member Type Code': str, 'member_type': str, 'Member Type': str, 'registration_type_code': str, 'Registration Type Code': str, 'registration_type': str, 'Registration Type': str, 'other_1': str, 'Other 1': str, 'other_2': str, 'Other 2': str, 'ticket_1_code': str, 'Ticket 1 Code': str, 'ticket_2_code': str, 'Ticket 2 Code': str, 'ticket_3_code': str, 'Ticket 3 Code': str, 'ticket_4_code': str, 'Ticket 4 Code': str, 'ticket_5_code': str, 'Ticket 5 Code': str, 'ticket_6_code': str, 'Ticket 6 Code': str, 'ticket_7_code': str, 'Ticket 7 Code': str, 'ticket_8_code': str, 'Ticket 8 Code': str, 'allow_tracking': str, 'Allow Tracking': str, 'agree_to_tc': str, 'Agree to TC': str, } ) df.rename(columns={ 'External ID': 'external_id', 'External Event ID': 'external_event_id', 'External Registration ID': 'external_registration_id', 'External Person ID': 'external_person_id', 'External Sys ID': 'external_person_id', 'Event Badge Template ID': 'event_badge_template_id', 'Pronouns': 'pronouns', 'Informal_name': 'informal_name', 'Title Names': 'title_names', 'Given Name': 'given_name', 'Middle Name': 'middle_name', 'Family Name': 'family_name', 'Designations': 'designations', 'Professional Title': 'professional_title', 'Full Name': 'full_name', 'Display Name': 'full_name_override', 'Full Name Override': 'full_name_override', 'Affiliations': 'affiliations', # 'Title': 'presenter_title_names', # 'Prefix': 'presenter_title_names', # 'Nickname': 'presenter_informal_name', # 'given_name (first)': 'presenter_given_name', # 'First Name': 'presenter_given_name', # 'family_name (last)': 'presenter_family_name', # 'Last Name': 'presenter_family_name', # 'Suffix': 'presenter_designations', 'Email': 'email', 'Email Address': 'email', 'CC Email Address': 'cc_email', 'Phone': 'phone', 'Location': 'location', 'Badge Type Code': 'badge_type_code', 'Badge Type': 'badge_type', 'Member Type Code': 'member_type_code', 'Member Type': 'member_type', 'Registration Type Code': 'registration_type_code', 'Registration Type': 'registration_type', 'Other 1': 'other_1', 'Other 2': 'other_2', 'Ticket 1 Code': 'ticket_1_code', 'Ticket 2 Code': 'ticket_2_code', 'Ticket 3 Code': 'ticket_3_code', 'Ticket 4 Code': 'ticket_4_code', 'Ticket 5 Code': 'ticket_5_code', 'Ticket 6 Code': 'ticket_6_code', 'Ticket 7 Code': 'ticket_7_code', 'Ticket 8 Code': 'ticket_8_code', 'Allow Tracking': 'allow_tracking', 'Agree to TC': 'agree_to_tc', # 'location_title': 'location_name', # 'Location Code': 'location_code', # 'Location Name': 'location_name', # 'session_location': 'location_name', # 'Session Location': 'location_name', # 'session_title': 'session_name', # 'Session Code': 'session_code', # 'Session Name': 'session_name', # 'presentation_title': 'presentation_name', # 'Presentation Code': 'presentation_code', # 'Presentation Name': 'presentation_name', # 'Presenter Code': 'code', # 'Presenter Number': 'number', # for sorting # 'Presenter Name': 'name', }, inplace = True) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(df) df_dict = df.to_dict(orient='records') log.info(f'Total record count: {len(df_dict)}') loop_count = 0 event_badge_person_li = [] event_badge_person_summary_li = [] log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL for record in df_dict: # ### Figure out the external IDs log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break if external_id := record.get('external_id'): log.info('Using given external_id for external_id') elif record.get('external_registration_id') and record.get('external_person_id'): log.info('Using external_registration_id and external_person_id for external_id') external_id = str(record.get('external_registration_id'))+':'+str(record.get('external_person_id')) elif record.get('external_event_id') and record.get('external_person_id'): log.info('Using external_event_id and external_person_id for external_id') external_id = str(record.get('external_event_id'))+':'+str(record.get('external_person_id')) elif record.get('external_person_id'): log.info('Using only external_person_id for external_id') external_id = str(record.get('external_person_id')) elif record.get('external_registration_id'): log.info('Using only external_registration_id for external_id') external_id = str(record.get('external_registration_id')) else: log.warning('No external ID was found or could safely be generated.') break log.debug(f'Event Badge External ID: {external_id}') event_person_summary = {} event_person_summary['event_id'] = event_id event_person_summary['event_id_random'] = event_id_random event_person_summary['external_id'] = external_id event_person_summary['given_name'] = record.get('given_name') event_person_summary['family_name'] = record.get('family_name') event_person_summary['email'] = record.get('email') event_person_data = {} event_person_data['account_id'] = account_id # Is this needed? event_person_data['event_id'] = event_id event_person_data['enable'] = True event_person_data['external_id'] = external_id event_person_data['external_event_id'] = record.get('external_event_id') event_person_data['external_registration_id'] = record.get('external_registration_id') # event_person_data['external_reg_id'] = record.get('external_registration_id') # Deprecated event_person_data['external_person_id'] = record.get('external_person_id') # event_person_data['external_sys_id'] = record.get('external_person_id') # Deprecated event_person_data['event_badge'] = {} event_person_data['event_person_profile'] = {} # event_person_data['event_registration'] = {} # Not currently used event_person_data['event_person_profile']['enable'] = True event_person_data['event_person_profile']['pronouns'] = record.get('pronouns') event_person_data['event_person_profile']['informal_name'] = record.get('informal_name') event_person_data['event_person_profile']['title_names'] = record.get('title_names') event_person_data['event_person_profile']['given_name'] = record.get('given_name') event_person_data['event_person_profile']['family_name'] = record.get('family_name') event_person_data['event_person_profile']['designations'] = record.get('designations') event_person_data['event_person_profile']['professional_title'] = record.get('professional_title') event_person_data['event_person_profile']['full_name'] = record.get('full_name') event_person_data['event_person_profile']['affiliations'] = record.get('affiliations') event_person_data['event_person_profile']['email'] = record.get('email') event_person_data['event_person_profile']['phone'] = record.get('phone') event_person_data['event_person_profile']['location'] = record.get('location') event_person_data['event_badge']['external_id'] = external_id event_person_data['event_badge']['external_event_id'] = record.get('external_event_id') event_person_data['event_badge']['external_registration_id'] = record.get('external_registration_id') # event_person_data['event_badge']['external_reg_id'] = record.get('external_registration_id') # Deprecated event_person_data['event_badge']['external_person_id'] = record.get('external_person_id') # event_person_data['event_badge']['external_sys_id'] = record.get('external_person_id') # Deprecated event_person_data['event_badge']['enable'] = True event_person_data['event_badge']['pronouns'] = record.get('pronouns') event_person_data['event_badge']['informal_name'] = record.get('informal_name') event_person_data['event_badge']['title_names'] = record.get('title_names') event_person_data['event_badge']['given_name'] = record.get('given_name') event_person_data['event_badge']['family_name'] = record.get('family_name') event_person_data['event_badge']['designations'] = record.get('designations') event_person_data['event_badge']['professional_title'] = record.get('professional_title') event_person_data['event_badge']['full_name'] = record.get('full_name') event_person_data['event_badge']['affiliations'] = record.get('affiliations') event_person_data['event_badge']['email'] = record.get('email') event_person_data['event_badge']['phone'] = record.get('phone') event_person_data['event_badge']['location'] = record.get('location') event_person_data['event_badge']['event_badge_template_id'] = 9 # record.get('event_badge_template_id') event_person_data['event_badge']['badge_type_code'] = record.get('badge_type_code') event_person_data['event_badge']['badge_type'] = record.get('badge_type') # email = None # city = None # country_subdivision_code = None # state_province = None # state_province_abb = None # country_alpha_2_code = None # country = None # city = record.get('city') # state_province = record.get('state_province') # country = record.get('country') # country_alpha_2_code = record.get('country_alpha_2_code') # if location := record.get('location'): # else: # location = f'{city} {state_province} {country_alpha_2_code}' event_badge_person_li.append(event_person_data) event_badge_person_summary_li.append(event_person_summary) # event_person_summary_li.append(event_person_summary_data) loop_count = loop_count + 1 sql_select_event_person = f""" SELECT id AS event_person_id, id_random AS event_person_id_random, external_id AS event_person_external_id, external_event_id AS event_person_external_event_id, external_registration_id AS event_person_external_registration_id, external_person_id AS event_person_external_person_id, event_badge_id AS event_badge_id, event_person_profile_id AS event_person_profile_id, event_registration_id AS event_registration_id FROM `event_person` AS `event_person` WHERE event_person.event_id = :event_id AND event_person.external_id = :external_id /*LIMIT 1*/; """ if event_person_result := sql_select(sql=sql_select_event_person, data=event_person_summary): if isinstance(event_person_result, list): log.error(f'Found more than one Event Person with the same External ID. Count: {len(event_person_result)}') # return False else: event_person_id = event_person_result.get('event_person_id') event_badge_id = event_person_result.get('event_badge_id') event_person_profile_id = event_person_result.get('event_person_profile_id') log.info(f'Found Event Person. Updating existing... Event Person ID: {event_person_id}') if create_event_person_obj_result := create_update_event_person_obj_v4( event_person_dict_obj = event_person_data, event_person_id = event_person_id, account_id = account_id, event_id = event_id, event_badge_id = event_badge_id, event_person_profile_id = event_person_profile_id, # create_sub_obj = create_sub_obj, # fail_any = fail_any, # return_outline = False, ): event_person_id = create_event_person_obj_result log.warning(f'Event Person updated. Event Person ID: {event_person_id}') else: log.warning(f'Event Person not updated. Event Person ID: {event_person_id}') log.debug(event_badge_obj_in_result) # return False else: log.info('No Event Person found. Creating new...') if create_event_person_obj_result := create_update_event_person_obj_v4( event_person_dict_obj = event_person_data, account_id = account_id, event_id = event_id, # create_sub_obj = create_sub_obj, # fail_any = fail_any, # return_outline = False, ): event_person_id = create_event_person_obj_result log.warning(f'Event Person created. Event Person ID: {event_person_id}') else: log.warning(f'Event Person not created.') log.debug(event_badge_obj_in_result) # return False if return_detail: return mk_resp(data=event_badge_person_li, status_message=f'Importing badges from file. Found {len(person_li)} badges.', response=commons.response) else: return mk_resp(data=event_badge_person_summary_li, status_message=f'Checked for badges from file. Found {len(event_badge_person_li)} badges.', response=commons.response)