import datetime, json, pytz, secrets, time import pandas, xlrd # qrcode from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Response, status from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string, common_route_params, Common_Route_Params from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.contact_methods import load_contact_obj, update_contact_obj from app.methods.person_methods import create_update_person_obj_v4b, get_person_rec_list, load_person_obj from app.methods.user_methods import load_user_obj from app.models.contact_models import Contact_Base from app.models.person_models import Person_Base from app.models.user_models import User_Base, User_New_Base, User_Out_Base from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() ext_id_update_list = [ 'MO69EB82~kstephan64@gmail.com', 'ON67FX34~k8maguire@gmail.com', 'MT62VR50~sdyanofsky@aol.com', 'TF39YD57~clcochran92@gmail.com', 'UY21HA29~drpsuski@gmail.com', 'TB98GM51~amandalh89@gmail.com', 'YY36MS82~paul@hopebiomed.us', 'QJ96XK21~abjackson1103@gmail.com', 'YE37SQ11~youellette@charter.net', 'MI18UY96~Cynthia.allyse@gmail.com', 'HI16RE87~Jamwen@comcast.net', 'XF70ZW11~oversteerdriver@gmail.com', 'BF75WK45~will@southtampapsychiatry.com', 'PR72VQ78~enge0368@gmail.com', 'JL24HM67~Fpmedicalconsulting@gmail.com', 'TH94DZ12~gomezjonathandaniel@gmail.com', 'BF33UZ69~tdrake38@icloud.com', 'ET63JU61~Drjessicabokelman@gmail.com', 'YN81XM59~Mrsbeason01@gmail.com', 'JQ73UM57~anthonyjgrandelis@gmail.com', 'HR75SP80~pnorris@med.miami.edu', 'SN97HZ72~ajmiglic@gmail.com', 'RA21CS48~Jacob.goldenberg22393@gmail.com', 'NO28OF94~Kathymrs65@gmail.com', 'IE25KZ46~stremickshannon@gmail.com', 'XK22WI84~skmcardinal11@gmail.com', 'ZO75WD68~mail@zensis.org', 'DE73VS79~doccarlos123@gmail.com', 'HT32CM64~marchese_31@yahoo.com', 'WM97PX89~jeff@jeff-hirsch.com', 'NJ32FW76~Louisearle@gmail.com', 'XU23HP89~louissolis@hotmail.com', 'TN90TO83~Ryan.d.odonnell@gmail.com', 'YW91KD57~Wijekoon@gmail.com', 'NR52NF96~Drjanehart@gmail.com', 'XE18DK68~mackenzie.tray@gmail.com', 'LD88CR61~cwsurgeon69@gmail.com', 'HN93VC10~lepricano1@gmail.com', 'PX67ST57~poojanparikh@outlook.com', 'IP16BM33~fgoldberg@nlh.org', 'DF87TA51~Nicolelfrost@gmail.com', 'HY24IL79~tnelsonmd@yahoo.com', ] # Based on the Cvent Address Book Contacts export data with new External IDs added as needed. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-04 @router.get('/update_w_external_id', response_model=Resp_Body_Base) async def importing_update_w_external_id( response: Response = Response, ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 13 full_file_path = 'admin/temp/import_person_external_id.csv' log.info(f'Starting import for Account ID {account_id} File Name: {full_file_path}') df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str}) log.debug(df) df_dict = df.to_dict(orient='records') person_data_li = [] for record in df_dict: person_id = None if external_id := record.get('external_id', None): pass elif external_id := record.get('External ID', None): pass else: log.info('No external ID was found.') continue if source_id := record.get('source_id', None): pass elif source_id := record.get('Source ID', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass elif email := record.get('Email Address', None): pass else: log.info('No email address was found.') continue log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') data = {} data['account_id'] = account_id data['email'] = email sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.email = :email AND (external_id IS NULL OR external_sys_id IS NULL) LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record person_rec = person_rec_result person_id = person_rec.get('person_id', None) person_id_random = person_rec.get('person_id_random', None) full_name = person_rec.get('full_name', None) log.warning(f'Found one record. Person ID: {person_id_random} Person Full Name: {full_name}') # contact_id = person_rec.get('contact_id', None) # address_id = person_rec.get('address_id', None) # user_id = person_rec.get('user_id', None) # external_id = person_rec.get('external_id', None) # external_sys_id = person_rec.get('external_sys_id', None) person_data = {} person_data['id'] = person_id person_data['external_id'] = external_id if source_id: person_data['external_sys_id'] = source_id else: person_data['external_sys_id'] = email log.debug(person_data) if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.info(f'No record found that needs to be updated. Email: {email}') continue person_data_min = {} person_data_min['person_id'] = person_id person_data_min['full_name'] = person_rec.get('full_name', None) person_data_min['email'] = email person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}") return mk_resp(data=person_data_li, response=response) # Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function. # All records should have an External ID. # Updated 2021-10-04 @router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base) async def ins_up_person_contact_address_user_data( begin_at: int = 0, end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_insert_person = True allow_insert_contact = True allow_insert_address = True allow_insert_user = True # allow_insert_membership = True allow_update_person = True allow_update_contact = True allow_update_address = True allow_update_user = True # allow_update_membership = True # allow_update_person_new_user = True account_id = 13 full_file_path = 'admin/temp/import_person_contact_address_user_data.csv' # full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx' df = pandas.read_csv( full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) df.rename(columns={ 'Source ID': 'source_id', 'External ID': 'external_id', 'Contact Type': 'contact_type', 'Email Address': 'email', 'Nickname': 'informal_name', 'Prefix': 'name_prefix', 'First Name': 'given_name', 'Middle Name': 'middle_name', 'Last Name': 'family_name', 'Suffix': 'name_suffix', 'Title': 'title_unknown', 'Company': 'company_name', # affiliations 'CC Email Address': 'cc_email', 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in 'Convention History': 'convention_history', 'Date of Birth': 'date_of_birth', 'Non-Binary Gender': 'gender_non-binary', 'Gender': 'gender', 'Mobile Phone': 'phone_mobile', 'Home Phone': 'phone_home', 'Home Fax': 'phone_home_fax', 'Home Address 1': 'home_address_line_1', 'Home Address 2': 'home_address_line_2', 'Home Address 3': 'home_address_line_3', 'Home City': 'home_address_city', 'Home State': 'home_address_state_province_code', 'Home State Name': 'home_address_state_province_name', 'Home Country': 'home_address_country_name', # home_address_country_code 'Home ZIP/Postal Code': 'home_address_postal_code', 'Primary Address': 'primary_address', 'Membership Type': 'membership_type', 'Membership Join Date': 'membership_type_join_date', 'Membership Last Renewal Date': 'membership_type_last_renewal_date', 'Membership Expiration Date': 'membership_type_expiration_date', 'Opted-Out': 'email_opt_out', 'Paper Mail Opted Out': 'paper_mail_opt_out', 'I am joining IDAA as a': 'joining_as', 'Practice Specialty': 'practice_specialty', 'Practice Status': 'practice_status', 'Designation': 'designation', 'Professional Designations': 'professional_designations', 'Qualifying Degree': 'qualifying_degree', 'Recovery Programs': 'recovery_program', 'Scholarship History': 'scholarship_history', 'Sobriety Date': 'sobriety_date', 'Sobriety Date - data from old system': 'sobriety_date_old_sys', 'Emergency Contact': 'emergency_contact', 'SO Name Prefix': 'so_name_prefix', 'SO First Name': 'so_name_given_name', 'SO Last Name': 'so_name_family_name', 'SO Professional Designation': 'so_professional_designation', 'Guest Home City': 'guest_home_address_city', 'Guest Home State/Province Code': 'guest_home_address_state_province_code', 'Work Phone': 'phone_work', 'Work Fax': 'phone_work_fax', 'Work Address 1': 'work_address_line_1', 'Work Address 2': 'work_address_line_2', 'Work Address 3': 'work_address_line_3', 'Work City': 'work_address_city', 'Work State': 'work_address_state_province_code', 'Work State Name': 'work_address_state_province_name', 'Work ZIP/Postal Code': 'work_address_postal_code', 'Work Country': 'work_address_country_code', 'Created By': 'created_by_name', 'Creation Date': 'created_by_date', }, inplace = True) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) loop_count = 0 person_data_li = [] # for i in df.index: for record in df_dict: log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break person_new = None person_id = None contact_id = None address_id = None user_id = None # person_profile_id = None # membership_person_id = None if external_id := record.get('external_id', None): pass else: log.warning('No external ID was found.') continue # This block should generally not be needed -Scott 2021-12-17 if external_id in ext_id_update_list: log.info('In list') pass else: # log.info('Not in list') continue if source_id := record.get('source_id', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass else: log.warning('No email address was found.') continue log.info(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') person_data = {} person_data['external_id'] = external_id if source_id: person_data['external_sys_id'] = source_id else: person_data['external_sys_id'] = email external_sys_id = person_data['external_sys_id'] log.debug(external_sys_id) if record.get('informal_name'): person_data['informal_name'] = record.get('informal_name') if record.get('given_name'): person_data['given_name'] = record.get('given_name') if record.get('middle_name'): person_data['middle_name'] = record.get('middle_name') if record.get('family_name'): person_data['family_name'] = record.get('family_name') if record.get('name_prefix'): person_data['title_names'] = record.get('name_prefix') if record.get('name_suffix'): person_data['designations'] = record.get('name_suffix') if record.get('informal_name'): person_data['informal_full_name'] = record.get('informal_name') if record.get('informal_name'): person_data['informal_display_name'] = record.get('informal_name') if person_data.get('given_name') and person_data.get('middle_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] elif person_data.get('given_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['family_name'] elif person_data.get('given_name'): full_name = person_data['family_name'] elif record.get('informal_full_name'): full_name = record['informal_full_name'] elif record.get('informal_name'): full_name = record['informal_name'] else: full_name = record.get('email') # if record.get('informal_full_name', None): # person_data['informal_full_name'] = record['informal_full_name'] # else: # person_data['informal_full_name'] = None # person_data['last_first_name'] = record.get('last_first_name', None) if designation := record.get('designation', ''): # This field is mostly empty in IDAA data from Cvent if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip()[:125] # professional designation else: person_data['designations'] = str(designation) # professional designation elif designation := record.get('professional_designations', ''): if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip()[:125] # professional designation else: person_data['designations'] = str(designation)[:125] # professional designation if birth_date := record.get('date_of_birth'): person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y').date() if gender_name := record.get('gender_non-binary'): if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 if gender_name == 'Male': person_data['lu_gender_id'] = 2 if gender_name == 'Female': person_data['lu_gender_id'] = 3 if gender_name == 'Non-binary/third gender': person_data['lu_gender_id'] = 4 if email_opt_out := person_data.get('email_opt_out'): if email_opt_out == 'Yes': person_data['email_allowed'] = False if email_opt_out == 'No': person_data['email_allowed'] = True if paper_mail_opt_out := person_data.get('paper_mail_opt_out'): if paper_mail_opt_out == 'Yes': person_data['paper_mail_allowed'] = False if paper_mail_opt_out == 'No' or paper_mail_opt_out == 0: person_data['paper_mail_allowed'] = True person_data['enable'] = True if created_on := record.get('created_on'): person_data['created_on'] = datetime.datetime.strptime(created_on, '%d-%b-%Y') if updated_on := record.get('updated_on'): person_data['updated_on'] = datetime.datetime.strptime(updated_on, '%d-%b-%Y') other_data = {} other_data['contact_type'] = record.get('contact_type', None) other_data['membership_type'] = record.get('membership_type', None) other_data['membership_join_date'] = record.get('membership_join_date', None) other_data['membership_expiration_date'] = record.get('membership_expiration_date', None) other_data['membership_last_renewal_date'] = record.get('membership_last_renewal_date', None) other_data['convention_history'] = record.get('convention_history', None) meta_data = {} meta_data['created_by_method'] = record.get('created_by_method', None) meta_data['created_by_date'] = record.get('created_by_date', None) meta_data['created_by_name'] = record.get('created_by_name', None) meta_data['modified_by'] = record.get('modified_by', None) person_data['other_json'] = json.dumps(other_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4) # Look up by email address or external ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed # Process the person data log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue data = {} data['account_id'] = account_id data['external_id'] = external_id data['external_sys_id'] = external_sys_id # sql = f""" # SELECT * # FROM `v_person` AS `person` # WHERE person.account_id = :account_id # AND person.external_sys_id = :external_sys_id # LIMIT 1; # """ sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_id = :external_id LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record log.debug(person_rec_result) person_rec = person_rec_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id', None) address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) # import_new_person(person_id=person_id, contact_id=contact_id, address=address_id, user_id=user_id) person_data['id'] = person_id log.info(f'Found Person ID: {person_id} Contact ID: {contact_id} Address ID: {address_id} User ID: {user_id}') if 'created_on' in person_data: person_data.pop('created_on') # I don't want to reset the created date? if 'updated_on' in person_data: person_data.pop('updated_on') # I don't want to reset the updated date? # person_data.pop('other_json') # person_data.pop('meta_json') if allow_update_person: if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Person updating not allowed') # Do nothing continue else: # INSERT new record log.debug('Found no records or something went wrong. Creating a new record...') person_data['account_id'] = account_id if allow_insert_person: if person_obj_in_result := sql_insert(data=person_data, table_name='person'): log.debug(person_obj_in_result) person_id = person_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(person_obj_in_result) continue # Something unexpected may have happened else: log.warning('Person inserting not allowed') # Do nothing continue # Process the contact data log.info('Process the contact data') contact_data = {} contact_data['email'] = record['email'] if record.get('email_status', None): contact_data['email_status'] = record['email_status'] if record('email_status', None) != 'Undeliverable': contact_data['email_active'] = True else: contact_data['email_active'] = False # else: # contact_data['email_active'] = None # pass # if record('email_status', None): # contact_data['email_status'] = record['email_status'] # else: # contact_data['email_status'] = None if cc_email := record.get('cc_email', None): contact_data['cc_email'] = cc_email if record['phone_mobile']: contact_data['phone_mobile'] = record['phone_mobile'] else: contact_data['phone_mobile'] = None if record['phone_home']: contact_data['phone_home'] = record['phone_home'] else: contact_data['phone_home'] = None if record['phone_home_fax'] and record['phone_work_fax']: contact_data['phone_fax'] = record['phone_home_fax'] contact_data['phone_other'] = record['phone_work_fax'] elif record['phone_home_fax']: contact_data['phone_fax'] = record['phone_home_fax'] elif record['phone_work_fax']: contact_data['phone_fax'] = record['phone_work_fax'] if record['phone_work']: contact_data['phone_office'] = record['phone_work'] else: contact_data['phone_office'] = None if record.get('facebook_url'): contact_data['facebook_url'] = record.get('facebook_url') if record.get('linkedin_url'): contact_data['linkedin_url'] = record.get('linkedin_url') if record.get('twitter_url'): contact_data['twitter_url'] = record.get('twitter_url') contact_data['enable'] = True log.debug(contact_data) if contact_id: # UPDATE existing contact record log.info('UPDATE existing contact record') contact_data['id'] = contact_id if allow_update_contact: if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): log.debug(contact_obj_up_result) else: log.warning(contact_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new contact record and link to person record log.info('INSERT new contact record and link to person record') contact_data['account_id'] = account_id contact_data['for_type'] = 'person' contact_data['for_id'] = person_id if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): log.debug(contact_obj_in_result) contact_id = contact_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(contact_obj_in_result) continue # Something unexpected may have happened # Process the contact address data log.info('Process the contact address data') address_data = {} if record['home_address_line_1']: address_data['line_1'] = record['home_address_line_1'] else: address_data['line_1'] = None if record['home_address_line_2']: address_data['line_2'] = record['home_address_line_2'] else: address_data['line_2'] = None if record['home_address_line_3']: address_data['line_3'] = record['home_address_line_3'] else: address_data['line_3'] = None if record['home_address_city']: address_data['city'] = record['home_address_city'] else: address_data['city'] = None if record.get('home_address_country_code', None) and record.get('home_address_state_province_code', None): address_data['country_subdivision_code'] = record['home_address_country_code']+'-'+record['home_address_state_province_code'] else: address_data['country_subdivision_code'] = None if record.get('home_address_state_province_name', None): address_data['state_province'] = record.get('home_address_state_province_name', None) if record['home_address_postal_code']: address_data['postal_code'] = record['home_address_postal_code'] else: address_data['postal_code'] = None if record.get('home_address_country_code', None): address_data['country_alpha_2_code'] = record.get('home_address_country_code', None) else: address_data['country_alpha_2_code'] = None if record.get('home_address_country_name', None): address_data['country'] = record.get('home_address_country_name', None) address_data['enable'] = True log.debug(address_data) if address_id: # UPDATE existing address record log.info('UPDATE existing address record') address_data['id'] = address_id if allow_update_address: if address_obj_up_result := sql_update(data=address_data, table_name='address'): log.debug(address_obj_up_result) else: log.warning(address_obj_up_result) # continue # Something unexpected may have happened elif contact_id: # INSERT new address record and link to contact record log.info('INSERT new address record and link to contact record') address_data['account_id'] = account_id address_data['for_type'] = 'contact' address_data['for_id'] = contact_id if address_obj_in_result := sql_insert(data=address_data, table_name='address'): log.debug(address_obj_in_result) address_id = address_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(address_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No address ID to update or contact ID to create an address linked to the contact.') return False # Process the user data log.info('Process the user data') user_data = {} user_data['name'] = full_name # person_data['full_name'] user_data['username'] = record['email'] user_data['email'] = record['email'] user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start user_data['enable'] = True user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) user_data['super'] = False user_data['manager'] = False # user_data['administrator'] = False user_data['public'] = False user_data['verified'] = True user_data['notes'] = 'Created by importing list' log.debug(user_data) if user_id: log.info('UPDATE existing user record') user_data['id'] = user_id user_data.pop('enable') user_data.pop('email_verified') # user_data.pop('administrator') user_data.pop('notes') log.debug(user_data) if allow_update_user: if user_obj_up_result := sql_update(data=user_data, table_name='user'): log.debug(user_obj_up_result) else: log.warning(user_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new user record and link to person record log.info('INSERT new user record and link to person record') user_data['account_id'] = account_id user_data['person_id'] = person_id random_password_string = secrets.token_urlsafe(8) user_data['password'] = secure_hash_string(string=random_password_string) other_data = {} other_data['temp_password'] = random_password_string user_data['other_json'] = json.dumps(other_data, indent=4) if allow_insert_user: if user_obj_in_result := sql_insert(data=user_data, table_name='user'): log.debug(user_obj_in_result) user_id = user_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(user_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No user ID to update or person ID to create a user linked to the person.') return False if person_new: log.debug('Updating person record one more time since this is a new person') person_data_up = {} person_data_up['id'] = person_id person_data_up['user_id'] = user_id # random_password_string # Don't need to update with the new contact or address IDs that were just created. if allow_insert_user: # Because this only matters on user INSERT if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) # break continue # Something unexpected may have happened person_data_min = {} person_data_min['person_id'] = person_id person_data_min['full_name'] = full_name person_data_min['email'] = email person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {full_name}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) # Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function. Person contact address user data needs to be updated. Use ins_up_person_contact_address_user_data() endpoint function. # All records should have an External ID. # Updated 2021-10-04 @router.get('/ins_up_membership_person_data', response_model=Resp_Body_Base) async def ins_up_membership_person_data( begin_at: int = 0, end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) log.info('Starting membership person data insert update...') allow_insert_person = True allow_insert_contact = True allow_insert_address = True allow_insert_user = True # allow_insert_membership = True allow_update_person = True allow_update_contact = True allow_update_address = True allow_update_user = True # allow_update_membership = True # allow_update_person_new_user = True account_id = 13 full_file_path = 'admin/temp/import_person_contact_address_user_data.csv' df = pandas.read_csv( full_file_path, na_filter=False, dtype={'external_id': str, 'External ID': str, 'source_id': str, 'Source ID': str, 'email': str, 'Email Address': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) df.rename(columns={ 'Source ID': 'source_id', 'External ID': 'external_id', 'Contact Type': 'contact_type', 'Email Address': 'email', 'Nickname': 'informal_name', 'Prefix': 'name_prefix', 'First Name': 'given_name', 'Middle Name': 'middle_name', 'Last Name': 'family_name', 'Suffix': 'name_suffix', 'Title': 'title_unknown', 'Company': 'company_name', # affiliations 'CC Email Address': 'cc_email', 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in 'Convention History': 'convention_history', 'Date of Birth': 'date_of_birth', 'Non-Binary Gender': 'gender_non-binary', 'Gender': 'gender', 'Mobile Phone': 'phone_mobile', 'Home Phone': 'phone_home', 'Home Fax': 'phone_home_fax', 'Home Address 1': 'home_address_line_1', 'Home Address 2': 'home_address_line_2', 'Home Address 3': 'home_address_line_3', 'Home City': 'home_address_city', 'Home State': 'home_address_state_province_code', 'Home State Name': 'home_address_state_province_name', 'Home Country': 'home_address_country_name', # home_address_country_code 'Home ZIP/Postal Code': 'home_address_postal_code', 'Primary Address': 'primary_address', 'Membership Type': 'membership_type', 'Membership Join Date': 'membership_type_join_date', 'Membership Last Renewal Date': 'membership_type_last_renewal_date', 'Membership Expiration Date': 'membership_type_expiration_date', 'Opted-Out': 'email_opt_out', 'Paper Mail Opted Out': 'paper_mail_opt_out', 'I am joining IDAA as a': 'joining_as', 'Practice Specialty': 'practice_specialty', 'Practice Status': 'practice_status', 'Designation': 'designation', 'Professional Designations': 'professional_designations', 'Qualifying Degree': 'qualifying_degree', 'Recovery Programs': 'recovery_program', 'Scholarship History': 'scholarship_history', 'Sobriety Date': 'sobriety_date', 'Sobriety Date - data from old system': 'sobriety_date_old_sys', 'Emergency Contact': 'emergency_contact', 'SO Name Prefix': 'so_name_prefix', 'SO First Name': 'so_name_given_name', 'SO Last Name': 'so_name_family_name', 'SO Professional Designation': 'so_professional_designation', 'Guest Home City': 'guest_home_address_city', 'Guest Home State/Province Code': 'guest_home_address_state_province_code', 'Work Phone': 'phone_work', 'Work Fax': 'phone_work_fax', 'Work Address 1': 'work_address_line_1', 'Work Address 2': 'work_address_line_2', 'Work Address 3': 'work_address_line_3', 'Work City': 'work_address_city', 'Work State': 'work_address_state_province_code', 'Work State Name': 'work_address_state_province_name', 'Work ZIP/Postal Code': 'work_address_postal_code', 'Work Country': 'work_address_country_code', 'Created By': 'created_by_name', 'Creation Date': 'created_by_date', }, inplace = True) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) loop_count = 0 person_data_li = [] for record in df_dict: log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break membership_new = None person_id = None contact_id = None address_id = None user_id = None # person_profile_id = None membership_person_id = None membership_person_type_id = None person_data = {} person_data['account_id'] = account_id if external_id := record.get('external_id', None): pass else: log.warning('No external ID was found.') continue # This block should generally not be needed -Scott 2021-12-17 if external_id in ext_id_update_list: log.info('In list') pass else: # log.info('Not in list') continue if source_id := record.get('source_id', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass else: log.info('No email address was found.') continue log.info(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') person_data = {} if source_id: person_data['external_sys_id'] = source_id else: person_data['external_sys_id'] = email external_sys_id = person_data['external_sys_id'] log.debug(external_sys_id) if membership_type_name := record.get('membership_type', None): pass else: log.warning(f'No membership information. Person ID: {person_id}, External ID: {external_id}, Email: {email}') continue data = {} data['account_id'] = account_id data['external_id'] = external_id data['external_sys_id'] = external_sys_id sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_id = :external_id /*AND person.external_sys_id = :external_sys_id*/ LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs of existing person record log.debug('Found one record') person_rec = person_rec_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id', None) address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) membership_person_id = person_rec.get('membership_person_id', None) log.info(f'Found membership_person_id: {membership_person_id}') else: log.warning(f'A person was not found with External ID: {external_id}') continue membership_person_data = {} membership_person_type_data = {} # datetime.datetime.strptime(date_time_str, '%m/%d/%Y') membership_person_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') # membership_person_type_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_data['first_start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_type_data['first_start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') if membership_type_last_renewal_date := record.get('membership_type_last_renewal_date', None): membership_person_data['start_on'] = datetime.datetime.strptime(membership_type_last_renewal_date, '%m/%d/%Y') membership_person_type_data['start_on'] = datetime.datetime.strptime(membership_type_last_renewal_date, '%m/%d/%Y') else: membership_person_data['start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_type_data['start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_data['end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_person_type_data['end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_person_data['last_end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_person_type_data['last_end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') current_datetime = datetime.datetime.now() if membership_person_data['end_on'] >= current_datetime: membership_person_data['lu_membership_person_status_id'] = 5 # 5 = active; expiration is > now membership_person_type_data['lu_membership_type_status_id'] = 5 # 5 = active; expiration is > now else: membership_person_data['lu_membership_person_status_id'] = 7 # 7 = inactive; expiration is < now membership_person_type_data['lu_membership_type_status_id'] = 7 # 7 = inactive; expiration is > now # membership_person_type_data['membership_person_id'] = membership_person_id if membership_type_name == 'Al-Anon Member' or membership_type_name == 'Al-Anon Members': membership_person_type_data['membership_type_id'] = 6 membership_person_type_data['product_id'] = 13 membership_person_data['level'] = 1 membership_person_type_data['level'] = 1 elif membership_type_name == 'Annual Contribution' or membership_type_name == 'Annual Contributions': # Unsure... making affiliate membership_person_type_data['membership_type_id'] = 8 membership_person_type_data['product_id'] = 13 membership_person_data['level'] = 3 membership_person_type_data['level'] = 3 elif membership_type_name == 'Doctoral Qualifying Member' or membership_type_name == 'Doctoral Qualifying Members': membership_person_type_data['membership_type_id'] = 5 membership_person_type_data['product_id'] = 4 membership_person_data['level'] = 1 membership_person_type_data['level'] = 1 elif membership_type_name == 'Student Member' or membership_type_name == 'Student Members': membership_person_type_data['membership_type_id'] = 7 membership_person_type_data['product_id'] = 14 membership_person_data['level'] = 1 membership_person_type_data['level'] = 1 membership_person_data['enable'] = True membership_person_type_data['enable'] = True if membership_person_id: # Update the membership records membership_person_data['id'] = membership_person_id # sql_update() # UPDATE membership_person log.info('UPDATE membership_person...') if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_up_result) else: log.warning(membership_person_obj_up_result) # continue # Something unexpected may have happened # sql_update() # UPDATE membership_person_type log.info('UPDATE membership_person_type...') data = {} data['membership_person_id'] = membership_person_id sql = f""" SELECT * FROM `v_membership_person` AS `membership_person` WHERE membership_person.id = :membership_person_id LIMIT 1; """ if membership_person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs of existing person record log.debug('Found one record') membership_person_rec = membership_person_rec_result membership_person_type_id = membership_person_rec.get('membership_person_type_id', None) log.info(f'Found membership_person_type_id: {membership_person_type_id}') else: continue membership_person_type_data['id'] = membership_person_type_id if membership_person_type_obj_up_result := sql_update(data=membership_person_type_data, table_name='membership_person_type'): log.debug(membership_person_type_obj_up_result) else: log.warning(membership_person_type_obj_up_result) continue # Something unexpected may have happened else: # Create new membership records membership_person_data['account_id'] = account_id membership_person_data['person_id'] = person_id membership_person_data['user_id'] = user_id # sql_insert() # INSERT new membership_person log.info('INSERT new membership_person...') if membership_person_obj_in_result := sql_insert(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_in_result) membership_person_id = membership_person_obj_in_result # Should be an int membership_person_new = True # Need to UPDATE this record after the membership type person data is processed else: log.warning(membership_person_obj_in_result) # continue # Something unexpected may have happened # sql_insert() # INSERT new membership_person_type link using membership_person_id log.info('INSERT new membership_person_type link using membership_person_id...') membership_person_type_data['membership_person_id'] = membership_person_id if membership_person_type_obj_in_result := sql_insert(data=membership_person_type_data, table_name='membership_person_type'): log.debug(membership_person_type_obj_in_result) membership_person_type_id = membership_person_type_obj_in_result # Should be an int membership_person_type_new = True # Need to UPDATE this record after the membership type person data is processed else: log.warning(membership_person_type_obj_in_result) # continue # Something unexpected may have happened membership_person_data['membership_person_type_id'] = membership_person_type_id # sql_update() # UPDATE membership_person with new membership_person_type_id log.info('UPDATE membership_person with new membership_person_type_id...') membership_person_data['id'] = membership_person_id if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_up_result) else: log.warning(membership_person_obj_up_result) continue # Something unexpected may have happened person_data_min = {} person_data_min['person_id'] = person_id person_data_min['membership_person_id'] = membership_person_id person_data_min['membership_person_type_id'] = membership_person_type_id person_data_min['membership_type_name'] = membership_type_name person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {record['email']} External ID: {external_id}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) @router.post('/person_data', response_model=Resp_Body_Base) async def importing_person_data( response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_insert_person = True allow_insert_contact = False allow_insert_address = False allow_insert_user = True allow_update_person = False allow_update_contact = False allow_update_address = False allow_update_user = False # allow_update_person_new_user = True account_id = 99 full_file_path = 'admin/temp/import_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) person_data_li = [] # for i in df.index: for record in df_dict: person_new = None person_id = None contact_id = None address_id = None user_id = None person_data = {} if record['external_sys_id']: person_data['external_sys_id'] = record['external_sys_id'] else: person_data['external_sys_id'] = record['email'] # person_data['informal_name'] = record['informal_name'] person_data['given_name'] = record['given_name'] if record['middle_name']: person_data['middle_name'] = record['middle_name'] else: person_data['middle_name'] = None if record['family_name']: person_data['family_name'] = record['family_name'] else: person_data['family_name'] = None # person_data['designations'] = record['prefix'] person_data['designations'] = record['suffix'] if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] elif person_data['given_name'] and person_data['family_name']: person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] elif person_data['given_name']: person_data['full_name'] = person_data['family_name'] elif record['informal_full_name']: person_data['full_name'] = record['informal_full_name'] elif record['informal_name']: person_data['full_name'] = record['informal_name'] else: person_data['full_name'] = None if record['informal_full_name']: person_data['informal_full_name'] = record['informal_full_name'] else: person_data['informal_full_name'] = None person_data['last_first_name'] = record['last_first_name'] person_data['created_on'] = record['created_on'] person_data['updated_on'] = record['updated_on'] other_data = {} other_data['contact_type'] = record['contact_type'] # ???? meta_data = {} meta_data['created_by_method'] = record['created_by_method'] meta_data['created_by_name'] = record['created_by_name'] meta_data['modified_by'] = record['modified_by'] meta_data['created_by_method'] = record['created_by_method'] person_data['other_json'] = json.dumps(other_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4) # Look up by email address or external import ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed # Process the person data log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if person_rec_li_result := sql_select(table_name='v_person', field_name='external_sys_id', field_value=person_data['external_sys_id']): if not isinstance(person_rec_li_result, list): # Pull out IDs and UPDATE existing person record log.debug('Found one record') person_rec = person_rec_li_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id_new', None) # Using _new from view until old contact_id is removed from person table address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) person_data['id'] = person_id if allow_update_person: if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(person_rec_li_result) # Do nothing continue # Something unexpected may have happened person_rec_li = person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') person_data['account_id'] = account_id if allow_insert_person: if person_obj_in_result := sql_insert(data=person_data, table_name='person'): log.debug(person_obj_in_result) person_id = person_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(person_obj_in_result) continue # Something unexpected may have happened # Process the contact data log.debug('Process the contact data') contact_data = {} contact_data['email'] = record['email'] if record['email_status'] != 'Undeliverable': contact_data['email_active'] = True else: contact_data['email_active'] = False contact_data['email_status'] = record['email_status'] if record['email_status']: contact_data['email_status'] = record['email_status'] else: contact_data['email_status'] = None if record['phone_mobile']: contact_data['phone_mobile'] = record['phone_mobile'] else: contact_data['phone_mobile'] = None if record['phone_home']: contact_data['phone_home'] = record['phone_home'] else: contact_data['phone_home'] = None if record['phone_fax']: contact_data['phone_fax'] = record['phone_fax'] elif record['phone_fax'] and record['phone_work_fax']: contact_data['phone_fax'] = record['phone_fax'] contact_data['phone_other'] = record['phone_work_fax'] if record['phone_work']: contact_data['phone_office'] = record['phone_work'] else: contact_data['phone_office'] = None log.debug(contact_data) if contact_id: # UPDATE existing contact record log.info('UPDATE existing contact record') contact_data['id'] = contact_id if allow_update_contact: if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): log.debug(contact_obj_up_result) else: log.warning(contact_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new contact record and link to person record log.info('INSERT new contact record and link to person record') contact_data['account_id'] = account_id contact_data['for_type'] = 'person' contact_data['for_id'] = person_id if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): log.debug(contact_obj_in_result) contact_id = contact_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(contact_obj_in_result) continue # Something unexpected may have happened # Process the contact address data log.debug('Process the contact address data') address_data = {} if record['address_line_1']: address_data['line_1'] = record['address_line_1'] else: address_data['line_1'] = None if record['address_line_2']: address_data['line_2'] = record['address_line_2'] else: address_data['line_2'] = None if record['address_line_3']: address_data['line_3'] = record['address_line_3'] else: address_data['line_3'] = None if record['address_city']: address_data['city'] = record['address_city'] else: address_data['city'] = None if record['address_state_province_code']: address_data['country_subdivision_code'] = record['address_country_code']+'-'+record['address_state_province_code'] else: address_data['country_subdivision_code'] = None if record['address_postal_code']: address_data['postal_code'] = record['address_postal_code'] else: address_data['postal_code'] = None if record['address_country_code']: address_data['country_alpha_2_code'] = record['address_country_code'] else: address_data['country_alpha_2_code'] = None log.debug(address_data) if address_id: # UPDATE existing address record log.info('UPDATE existing address record') address_data['id'] = address_id if allow_update_address: if address_obj_up_result := sql_update(data=address_data, table_name='address'): log.debug(address_obj_up_result) else: log.warning(address_obj_up_result) # continue # Something unexpected may have happened elif contact_id: # INSERT new address record and link to contact record log.info('INSERT new address record and link to contact record') address_data['account_id'] = account_id address_data['for_type'] = 'contact' address_data['for_id'] = contact_id if address_obj_in_result := sql_insert(data=address_data, table_name='address'): log.debug(address_obj_in_result) address_id = address_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(address_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No address ID to update or contact ID to create an address linked to the contact.') return False # Process the user data log.debug('Process the user data') user_data = {} user_data['name'] = person_data['full_name'] user_data['username'] = record['email'] user_data['email'] = record['email'] user_data['email_verified'] = contact_data['email_active'] # Not perfect, but a good start user_data['enable'] = False user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) user_data['super'] = False user_data['manager'] = False user_data['administrator'] = False user_data['public'] = False user_data['verified'] = True user_data['notes'] = 'Created by importing list' log.debug(user_data) if user_id: # UPDATE existing user record log.info('UPDATE existing user record') user_data['id'] = user_id if allow_update_user: if user_obj_up_result := sql_update(data=user_data, table_name='user'): log.debug(user_obj_up_result) else: log.warning(user_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new user record and link to person record log.info('INSERT new user record and link to person record') user_data['account_id'] = account_id user_data['person_id'] = person_id random_password_string = secrets.token_urlsafe(8) user_data['password'] = secure_hash_string(string=random_password_string) other_data = {} other_data['temp_password'] = random_password_string user_data['other_json'] = json.dumps(other_data, indent=4) if allow_insert_user: if user_obj_in_result := sql_insert(data=user_data, table_name='user'): log.debug(user_obj_in_result) user_id = user_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(user_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No user ID to update or person ID to create a user linked to the person.') return False if person_new: log.debug('Updating person record one more time since this is a new person') person_data_up = {} person_data_up['id'] = person_id person_data_up['user_id'] = user_id # random_password_string # Don't need to update with the new contact or address IDs that were just created. if allow_insert_user: # Because this only matters on user INSERT if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) # break continue # Something unexpected may have happened person_data_li.append(person_data) log.debug(f"Record processed: {person_id} {person_data['full_name']}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) @router.post('/cont_edu_cert_person_data', response_model=Resp_Body_Base) async def importing_cont_edu_cert_person_data( response: Response = Response, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 19 cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) cont_edu_cert_person_data_li = [] # for i in df.index: for record in df_dict: cont_edu_cert_person_new = None person_id = None user_id = None cont_edu_cert_person_id = None cont_edu_cert_person_data = {} cont_edu_cert_person_data['cont_edu_cert_id'] = cont_edu_cert_id cont_edu_cert_person_data['enable'] = True cont_edu_cert_person_data['email'] = record['email'] cont_edu_cert_person_data['given_name'] = record['given_name'] if record.get('middle_name', None): cont_edu_cert_person_data['middle_name'] = record['middle_name'] else: cont_edu_cert_person_data['middle_name'] = None if record['family_name']: cont_edu_cert_person_data['family_name'] = record['family_name'] else: cont_edu_cert_person_data['family_name'] = None if cont_edu_cert_person_data['given_name'] and cont_edu_cert_person_data['middle_name'] and cont_edu_cert_person_data['family_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['given_name']+' '+cont_edu_cert_person_data['middle_name']+' '+cont_edu_cert_person_data['family_name'] elif cont_edu_cert_person_data['given_name'] and cont_edu_cert_person_data['family_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['given_name']+' '+cont_edu_cert_person_data['family_name'] elif cont_edu_cert_person_data['given_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['family_name'] elif record.get('informal_full_name', None): cont_edu_cert_person_data['full_name'] = record['informal_full_name'] elif record['informal_name']: cont_edu_cert_person_data['full_name'] = record['informal_name'] else: cont_edu_cert_person_data['full_name'] = None if record.get('informal_full_name', None): cont_edu_cert_person_data['informal_full_name'] = record['informal_full_name'] else: cont_edu_cert_person_data['informal_full_name'] = None cont_edu_cert_person_data['last_first_name'] = record.get('last_first_name', None) cont_edu_cert_person_data['display_name'] = record.get('display_name', None) # cont_edu_cert_person_data['created_on'] = record['created_on'] # cont_edu_cert_person_data['updated_on'] = record['updated_on'] other_data = {} other_data['other_guest_of'] = record['other_guest_of'] other_data['other_guest_li'] = record['other_guest_li'] cont_edu_cert_person_data['other_json'] = json.dumps(other_data, indent=4) # Look up by email address and INSERT or UPDATE new cont_edu_cert_person record # Process the cont_edu_cert_person data log.debug(cont_edu_cert_person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if cont_edu_cert_person_rec_li_result := sql_select(table_name='v_cont_edu_cert_person', field_name='email', field_value=cont_edu_cert_person_data['email']): if not isinstance(cont_edu_cert_person_rec_li_result, list): # Pull out IDs and UPDATE existing cont_edu_cert_person record log.debug('Found one record') cont_edu_cert_person_rec = cont_edu_cert_person_rec_li_result # person_id = cont_edu_cert_person_rec.get('person_id', None) # user_id = cont_edu_cert_person_rec.get('user_id', None) cont_edu_cert_person_data['id'] = cont_edu_cert_person_id if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_up_result) else: log.warning(cont_edu_cert_person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(cont_edu_cert_person_rec_li_result) # Do nothing continue # Something unexpected may have happened cont_edu_cert_person_rec_li = cont_edu_cert_person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') cont_edu_cert_person_data['account_id'] = account_id if cont_edu_cert_person_obj_in_result := sql_insert(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_in_result) cont_edu_cert_person_id = cont_edu_cert_person_obj_in_result # Should be an int cont_edu_cert_person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(cont_edu_cert_person_obj_in_result) continue # Something unexpected may have happened # if cont_edu_cert_person_new: # log.debug('Updating person record one more time since this is a new person') # cont_edu_cert_person_data_up = {} # cont_edu_cert_person_data_up['id'] = person_id # cont_edu_cert_person_data_up['user_id'] = user_id # random_password_string # # Don't need to update with the new contact or address IDs that were just created. # if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data_up, table_name='cont_edu_cert_person'): # log.debug(cont_edu_cert_person_obj_up_result) # else: # log.warning(cont_edu_cert_person_obj_up_result) # # break # continue # Something unexpected may have happened cont_edu_cert_person_data_li.append(cont_edu_cert_person_data) log.debug(f"Record processed: {cont_edu_cert_person_id} {cont_edu_cert_person_data['full_name']}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=cont_edu_cert_person_data_li) @router.post('/cont_edu_cert_person_data_touch', response_model=Resp_Body_Base) async def importing_cont_edu_cert_person_data_touch( response: Response = Response, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 19 cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) cont_edu_cert_person_data_li = [] # for i in df.index: for record in df_dict: cont_edu_cert_person_new = None person_id = None user_id = None cont_edu_cert_person_id = None cont_edu_cert_person_data = {} # cont_edu_cert_person_data['cont_edu_cert_id'] = cont_edu_cert_id cont_edu_cert_person_data['enable'] = True cont_edu_cert_person_data['email'] = record['email'] other_data = {} other_data['last_event_date'] = '2021-08-01' other_data['other_guest_of'] = record['other_guest_of'] other_data['other_guest_li'] = record['other_guest_li'] cont_edu_cert_person_data['other_json'] = json.dumps(other_data, indent=4) # Look up by email address and INSERT or UPDATE new cont_edu_cert_person record # Process the cont_edu_cert_person data log.debug(cont_edu_cert_person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if cont_edu_cert_person_rec_li_result := sql_select(table_name='v_cont_edu_cert_person', field_name='email', field_value=cont_edu_cert_person_data['email']): if not isinstance(cont_edu_cert_person_rec_li_result, list): # Pull out IDs and UPDATE existing cont_edu_cert_person record # log.debug('Found one record') cont_edu_cert_person_rec = cont_edu_cert_person_rec_li_result cont_edu_cert_person_id = cont_edu_cert_person_rec.get('cont_edu_cert_person_id', None) log.info(cont_edu_cert_person_id) # person_id = cont_edu_cert_person_rec.get('person_id', None) # user_id = cont_edu_cert_person_rec.get('user_id', None) cont_edu_cert_person_data['id'] = cont_edu_cert_person_id if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): # log.debug(cont_edu_cert_person_obj_up_result) pass else: log.warning(cont_edu_cert_person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(cont_edu_cert_person_rec_li_result) # Do nothing continue # Something unexpected may have happened cont_edu_cert_person_rec_li = cont_edu_cert_person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') cont_edu_cert_person_data['account_id'] = account_id if cont_edu_cert_person_obj_in_result := sql_insert(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_in_result) cont_edu_cert_person_id = cont_edu_cert_person_obj_in_result # Should be an int cont_edu_cert_person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(cont_edu_cert_person_obj_in_result) continue # Something unexpected may have happened cont_edu_cert_person_data_li.append(cont_edu_cert_person_data) log.debug(f"Record processed: {cont_edu_cert_person_id}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL return mk_resp(data=cont_edu_cert_person_data_li) # Business Group on Health program import # Updated 2022-04-06 @router.get('/bgh_program_import', response_model=Resp_Body_Base) async def bgh_program_import( begin_at: int = 0, end_at: int = 100, # response: Response = Response, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = commons.x_account_id # 4 GpLf_bnywCs event_id = 1446 full_file_path = 'admin/temp/import_bgh_2022-04_program_data.csv' df = pandas.read_csv( full_file_path, na_filter = False, dtype = { 'session_code': str, 'Session Code': str, 'session_title': str, 'session_description': str, 'session_start_datetime': str, 'session_end_datetime': str, 'session_location': str, 'session_location_code': str, }, parse_dates = ['session_start_datetime', 'session_end_datetime'] ) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) df.rename(columns={ 'Presenter Code': 'presenter_code', 'Presentation Code': 'presentation_code', 'Session Code': 'session_code', 'session_title': 'session_name', 'Session Title': 'session_name', 'session_location_code': 'location_code', 'session_location': 'location_name', }, inplace = True) log.debug(df) # datetime.datetime.strptime(updated_on, '%d-%b-%Y') df_dict = df.to_dict(orient='records') log.debug(df_dict) from app.methods.event_session_methods import create_event_session_obj, create_update_event_session_obj_v4, load_event_session_obj, get_event_session_rec_list, update_event_session_obj_v3, update_event_session_obj from app.models.event_session_models import Event_Session_Base session_dict_li = [] for record in df_dict: session_dict = {} session_dict['code'] = record.get('session_code') session_dict['name'] = record.get('session_name') session_dict['description'] = record.get('session_description') session_dict['start_datetime'] = record.get('session_start_datetime') session_dict['end_datetime'] = record.get('session_end_datetime') location_name = record.get('location_name') if location_name == 'Broadway 1': session_dict['event_location_id'] = 301 elif location_name == 'Broadway 2': session_dict['event_location_id'] = 302 elif location_name == 'Broadway 3': session_dict['event_location_id'] = 303 elif location_name == 'Grand Ballroom': session_dict['event_location_id'] = 304 elif location_name == 'Sobro': session_dict['event_location_id'] = 305 create_update_event_session_obj_v4_result = create_update_event_session_obj_v4( event_session_dict_obj = session_dict, event_id = event_id ) log.debug(create_update_event_session_obj_v4_result) session_dict_li.append(session_dict) return mk_resp(data=session_dict_li, response=commons.response) # AAPOR program import # Updated 2022-04-11 @router.get('/aapor_program_import', response_model=Resp_Body_Base) async def aapor_program_import( event_id: str = Query(..., min_length=11, max_length=22), begin_at: int = 0, end_at: int = 50, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = commons.x_account_id # 20 j5EBhRDqPuw # AAPOR 2022: 1447 x2H2P2MYlXU if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass # elif event_exhibit_id is None: pass else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The Event ID was invalid or not found.') # PART 0: Load the data full_file_path = 'admin/temp/import_aapor_2022-04_program_data.csv' df = pandas.read_csv( full_file_path, na_filter = False, dtype = { 'presentation_code': str, 'presentation_title': str, 'presentation_code': str, 'presentation_start_datetime': str, 'presentation_end_datetime': str, 'presenter_number': str, 'presenter_given_name': str, 'presenter_family_name': str, 'session_code': str, 'Session Code': str, 'session_title': str, 'session_description': str, 'session_start_datetime': str, 'session_end_datetime': str, 'session_location': str, 'session_location_code': str, }, parse_dates = ['session_start_datetime', 'session_end_datetime'] ) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) df.rename(columns={ 'presenter_number': 'presenter_code', 'Presenter Number': 'presenter_code', 'Presenter Code': 'presenter_code', 'Presentation Code': 'presentation_code', 'presentation_title': 'presentation_name', 'given_name (first)': 'presenter_given_name', 'family_name (last)': 'presenter_family_name', 'Session Code': 'session_code', 'session_title': 'session_name', 'Session Title': 'session_name', 'session_location_code': 'location_code', 'session_location': 'location_name', }, inplace = True) log.debug(df) # datetime.datetime.strptime(updated_on, '%d-%b-%Y') df_dict = df.to_dict(orient='records') # log.debug(df_dict) from app.methods.event_session_methods import create_event_session_obj, create_update_event_session_obj_v4, load_event_session_obj, get_event_session_rec_list, update_event_session_obj_v3, update_event_session_obj from app.methods.event_presentation_methods import create_event_presentation_obj, create_update_event_presentation_obj_v4, load_event_presentation_obj, get_event_presentation_rec_list, update_event_presentation_obj_v3, update_event_presentation_obj from app.methods.event_presenter_methods import create_event_presenter_obj, create_update_event_presenter_obj_v4, load_event_presenter_obj, get_event_presenter_rec_list, update_event_presenter_obj_v3, update_event_presenter_obj from app.models.event_session_models import Event_Session_Base session_dict_li = [] # NOTE: I think I really like the way I did this loop counter and logging. -2022-04-14 loop_count = 0 for record in df_dict: loop_count = loop_count + 1 if loop_count <= begin_at: log.info(f'Skipping loop #{loop_count}') continue if loop_count > end_at: log.info(f'Stopping at loop #{loop_count}') break else: log.info(f'Starting loop #{loop_count}') log.info('# PART 1: Process the session data') session_data = {} session_data['event_id'] = event_id session_data['code'] = record.get('session_code') session_data['name'] = record.get('session_name') session_data['description'] = record.get('session_description') session_data['start_datetime'] = record.get('session_start_datetime') session_data['end_datetime'] = record.get('session_end_datetime') location_name = record.get('location_name') if location_name == 'Arkansas': session_data['event_location_id'] = 306 elif location_name == 'Colorado': session_data['event_location_id'] = 307 elif location_name == 'Erie': session_data['event_location_id'] = 308 elif location_name == 'Huron': session_data['event_location_id'] = 309 elif location_name == 'Michigan A': session_data['event_location_id'] = 310 elif location_name == 'Michigan B': session_data['event_location_id'] = 311 elif location_name == 'Mississippi': session_data['event_location_id'] = 312 elif location_name == 'Missouri': session_data['event_location_id'] = 313 elif location_name == 'Ontario': session_data['event_location_id'] = 314 elif location_name == 'Streeterville': session_data['event_location_id'] = 315 elif location_name == 'Superior A': session_data['event_location_id'] = 316 elif location_name == 'Superior B': session_data['event_location_id'] = 317 elif location_name == 'Wrigleyville': session_data['event_location_id'] = 318 session_data['enable'] = True session_data['hide'] = False log.debug(session_data) data = {} data['event_id'] = event_id data['code'] = session_data['code'] sql = f""" SELECT `event_session`.id AS 'event_session_id', `event_session`.id_random AS 'event_session_id_random' FROM `event_session` AS `event_session` WHERE event_session.event_id = :event_id AND event_session.code = :code LIMIT 1; """ # log.debug(sql) if event_session_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_session_rec = event_session_rec_li_result[0] log.debug(event_session_rec) else: # [] or False log.info('No results or something went wrong') event_session_rec = None log.debug(event_session_rec_li_result) # log.error('Hit break point') # break event_session_id = None if event_session_rec: log.info('Found a record. Updating...') event_session_id = event_session_rec.get('event_session_id') if update_event_session_obj_result := update_event_session_obj_v3( event_session_id = event_session_id, event_session_obj_exist = session_data ): pass else: log.error('Something went wrong while trying to update an event session.') # event_session_id = update_event_session_obj_result break log.debug(update_event_session_obj_result) else: log.info('No record found. Inserting...') if create_event_session_obj_result := create_event_session_obj( event_id = event_id, event_session_obj_new = session_data ): event_session_id = create_event_session_obj_result else: log.error('Something went wrong while trying to create a new event session.') # event_session_id = create_event_session_obj_result break log.debug(create_event_session_obj_result) # log.error('Hit continue to next loop point') # continue # log.error('Hit break point') # break log.info('# PART 2: Process the presentation data') presentation_data = {} presentation_data['for_type'] = 'event_session' presentation_data['for_id'] = event_session_id presentation_data['event_session_id'] = event_session_id presentation_data['code'] = record.get('presentation_code') presentation_data['name'] = record.get('presentation_name') presentation_data['description'] = record.get('presentation_description') presentation_data['start_datetime'] = record.get('presentation_start_datetime') presentation_data['end_datetime'] = record.get('presentation_end_datetime') presentation_data['enable'] = True session_data['hide'] = False log.debug(presentation_data) data = {} data['event_session_id'] = event_session_id data['code'] = presentation_data['code'] sql = f""" SELECT `event_presentation`.id AS 'event_presentation_id', `event_presentation`.id_random AS 'event_presentation_id_random' FROM `event_presentation` AS `event_presentation` WHERE event_presentation.event_session_id = :event_session_id AND event_presentation.code = :code LIMIT 1; """ log.debug(sql) if event_presentation_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_presentation_rec = event_presentation_rec_li_result[0] else: # [] or False log.info('No results or something went wrong') event_presentation_rec = None log.debug(event_presentation_rec_li_result) event_presentation_id = None if event_presentation_rec: log.info('Found a record. Updating...') event_presentation_id = event_presentation_rec.get('event_presentation_id') if update_event_presentation_obj_result := update_event_presentation_obj_v3( event_presentation_id = event_presentation_id, event_presentation_obj_exist = presentation_data, event_session_id = event_session_id, ): pass else: log.error('Something went wrong while trying to update an event presentation.') # event_presentation_id = update_event_presentation_obj_result break log.debug(update_event_presentation_obj_result) else: log.info('No record found. Inserting...') if create_event_presentation_obj_result := create_event_presentation_obj( event_session_id = event_session_id, event_presentation_obj_new = presentation_data ): event_presentation_id = create_event_presentation_obj_result else: log.error('Something went wrong while trying to create a new event presentation.') # event_presentation_id = create_event_presentation_obj_result break log.debug(create_event_presentation_obj_result) # log.error('Hit continue to next loop point') # continue # log.error('Hit break point') # break log.info('# PART 3: Process the presenter data') presenter_data = {} presenter_data['for_type'] = 'event_presentation' presenter_data['for_id'] = event_presentation_id presenter_data['event_session_id'] = event_session_id presenter_data['event_presentation_id'] = event_presentation_id presenter_data['code'] = record.get('presenter_code') presenter_data['given_name'] = record.get('presenter_given_name') presenter_data['family_name'] = record.get('presenter_family_name') presenter_data['sort'] = record.get('presenter_code') # Using the code as the sort as well presenter_data['enable'] = True presenter_data['hide'] = False log.debug(presenter_data) data = {} data['event_presentation_id'] = event_presentation_id data['code'] = presenter_data['code'] sql = f""" SELECT `event_presenter`.id AS 'event_presenter_id', `event_presenter`.id_random AS 'event_presenter_id_random' FROM `event_presenter` AS `event_presenter` WHERE event_presenter.event_presentation_id = :event_presentation_id AND event_presenter.code = :code LIMIT 1; """ log.debug(sql) if event_presenter_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_presenter_rec = event_presenter_rec_li_result[0] else: # [] or False log.info('No results or something went wrong') event_presenter_rec = None log.debug(event_presenter_rec_li_result) event_presenter_id = None if event_presenter_rec: log.info('Found a record. Updating...') event_presenter_id = event_presenter_rec.get('event_presenter_id') if update_event_presenter_obj_result := update_event_presenter_obj_v3( event_presenter_id = event_presenter_id, event_presenter_obj_exist = presenter_data, event_presentation_id = event_presentation_id, ): pass else: log.error('Something went wrong while trying to update an event presenter.') # event_presenter_id = update_event_presenter_obj_result break log.debug(update_event_presenter_obj_result) else: log.info('No record found. Inserting...') if create_event_presenter_obj_result := create_event_presenter_obj( event_presentation_id = event_presentation_id, event_presenter_obj_new = presenter_data ): event_presenter_id = create_event_presenter_obj_result else: log.error('Something went wrong while trying to create a new event presenter.') # event_presenter_id = create_event_presenter_obj_result break log.debug(create_event_presenter_obj_result) # log.error('Hit break point') # break return mk_resp(data=session_dict_li, response=commons.response) # CMSC program import # Updated 2022-05-10 @router.get('/cmsc_program_import', response_model=Resp_Body_Base) async def cmsc_program_import( event_id: str = Query(..., min_length=11, max_length=22), begin_at: int = 0, end_at: int = 50, commons: Common_Route_Params = Depends(common_route_params), ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = commons.x_account_id # 20 j5EBhRDqPuw # CMSC 2022: 1450 E4iJPaT7S8w if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The Event ID was invalid or not found.') # PART 0: Load the data full_file_path = 'admin/temp/import_cmsc_2022-06_program_data.csv' df = pandas.read_csv( full_file_path, na_filter = False, dtype = { 'presentation_code': str, 'presentation_title': str, 'presentation_code': str, 'presentation_start_datetime': str, 'presentation_end_datetime': str, 'presenter_number': str, 'presenter_given_name': str, 'presenter_family_name': str, 'session_code': str, 'Session Code': str, 'session_title': str, 'session_description': str, 'session_start_datetime': str, 'session_end_datetime': str, 'session_location': str, 'session_location_code': str, }, parse_dates = ['session_start_datetime', 'session_end_datetime'] ) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) # return str(df.info()) # return mk_resp(data=str(df.info()), status_code=500, response=commons.response, status_message='TESTING') df.rename(columns={ 'presenter_number': 'presenter_code', 'Presenter Number': 'presenter_code', 'Presenter Code': 'presenter_code', 'Presentation Code': 'presentation_code', 'presentation_title': 'presentation_name', 'given_name (first)': 'presenter_given_name', 'family_name (last)': 'presenter_family_name', 'Session Code': 'session_code', 'session_title': 'session_name', 'Session Title': 'session_name', 'session_location_code': 'location_code', 'session_location': 'location_name', }, inplace = True) log.debug(df) # datetime.datetime.strptime(updated_on, '%d-%b-%Y') df_dict = df.to_dict(orient='records') # log.debug(df_dict) from app.methods.event_session_methods import create_event_session_obj, create_update_event_session_obj_v4, load_event_session_obj, get_event_session_rec_list, update_event_session_obj_v3, update_event_session_obj from app.methods.event_presentation_methods import create_event_presentation_obj, create_update_event_presentation_obj_v4, load_event_presentation_obj, get_event_presentation_rec_list, update_event_presentation_obj_v3, update_event_presentation_obj from app.methods.event_presenter_methods import create_event_presenter_obj, create_update_event_presenter_obj_v4, load_event_presenter_obj, get_event_presenter_rec_list, update_event_presenter_obj_v3, update_event_presenter_obj from app.models.event_session_models import Event_Session_Base session_dict_li = [] # NOTE: I think I really like the way I did this loop counter and logging. -2022-04-14 loop_count = 0 for record in df_dict: loop_count = loop_count + 1 if loop_count <= begin_at: log.info(f'Skipping loop #{loop_count}') continue if loop_count > end_at: log.info(f'Stopping at loop #{loop_count}') break else: log.info(f'Starting loop #{loop_count}') log.info('# PART 1: Process the session data') session_data = {} session_data['event_id'] = event_id session_data['code'] = record.get('session_code') session_data['name'] = record.get('session_name') session_data['description'] = record.get('session_description') session_data['start_datetime'] = record.get('session_start_datetime') session_data['end_datetime'] = record.get('session_end_datetime') location_name = record.get('location_name') if location_name == 'Annapolis 1-4': session_data['event_location_id'] = 319 elif location_name == 'Potomac 1-3': session_data['event_location_id'] = 320 elif location_name == 'Potomac 4-6': session_data['event_location_id'] = 321 elif location_name == 'Potomac A': session_data['event_location_id'] = 322 elif location_name == 'Potomac A/B': session_data['event_location_id'] = 322 elif location_name == 'Potomac B': session_data['event_location_id'] = 323 elif location_name == 'Potomac C': session_data['event_location_id'] = 324 elif location_name == 'Potomac D': session_data['event_location_id'] = 325 elif location_name == 'Woodrow Wilson Ballroom B': session_data['event_location_id'] = 326 elif location_name == 'Woodrow Wilson Ballroom C': session_data['event_location_id'] = 327 elif location_name == 'Woodrow Wilson Ballroom D': session_data['event_location_id'] = 328 session_data['enable'] = True session_data['hide'] = False log.debug(session_data) data = {} data['event_id'] = event_id data['code'] = session_data['code'] sql = f""" SELECT `event_session`.id AS 'event_session_id', `event_session`.id_random AS 'event_session_id_random' FROM `event_session` AS `event_session` WHERE event_session.event_id = :event_id AND event_session.code = :code LIMIT 1; """ # log.debug(sql) if event_session_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_session_rec = event_session_rec_li_result[0] log.debug(event_session_rec) else: # [] or False log.info('No results or something went wrong') event_session_rec = None log.debug(event_session_rec_li_result) # log.error('Hit break point') # break event_session_id = None if event_session_rec: log.info('Found a record. Updating...') event_session_id = event_session_rec.get('event_session_id') if update_event_session_obj_result := update_event_session_obj_v3( event_session_id = event_session_id, event_session_obj_exist = session_data ): pass else: log.error('Something went wrong while trying to update an event session.') # event_session_id = update_event_session_obj_result break log.debug(update_event_session_obj_result) else: log.info('No record found. Inserting...') if create_event_session_obj_result := create_event_session_obj( event_id = event_id, event_session_obj_new = session_data ): event_session_id = create_event_session_obj_result else: log.error('Something went wrong while trying to create a new event session.') # event_session_id = create_event_session_obj_result break log.debug(create_event_session_obj_result) # log.error('Hit continue to next loop point') # continue # log.error('Hit break point') # break log.info('# PART 2: Process the presentation data') presentation_data = {} presentation_data['for_type'] = 'event_session' presentation_data['for_id'] = event_session_id presentation_data['event_session_id'] = event_session_id presentation_data['code'] = record.get('presentation_code') presentation_data['name'] = record.get('presentation_name') presentation_data['description'] = record.get('presentation_description') presentation_data['start_datetime'] = record.get('presentation_start_datetime') presentation_data['end_datetime'] = record.get('presentation_end_datetime') presentation_data['enable'] = True session_data['hide'] = False log.debug(presentation_data) data = {} data['event_session_id'] = event_session_id data['code'] = presentation_data['code'] sql = f""" SELECT `event_presentation`.id AS 'event_presentation_id', `event_presentation`.id_random AS 'event_presentation_id_random' FROM `event_presentation` AS `event_presentation` WHERE event_presentation.event_session_id = :event_session_id AND event_presentation.code = :code LIMIT 1; """ log.debug(sql) if event_presentation_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_presentation_rec = event_presentation_rec_li_result[0] else: # [] or False log.info('No results or something went wrong') event_presentation_rec = None log.debug(event_presentation_rec_li_result) event_presentation_id = None if event_presentation_rec: log.info('Found a record. Updating...') event_presentation_id = event_presentation_rec.get('event_presentation_id') if update_event_presentation_obj_result := update_event_presentation_obj_v3( event_presentation_id = event_presentation_id, event_presentation_obj_exist = presentation_data, event_session_id = event_session_id, ): pass else: log.error('Something went wrong while trying to update an event presentation.') # event_presentation_id = update_event_presentation_obj_result break log.debug(update_event_presentation_obj_result) else: log.info('No record found. Inserting...') if create_event_presentation_obj_result := create_event_presentation_obj( event_session_id = event_session_id, event_presentation_obj_new = presentation_data ): event_presentation_id = create_event_presentation_obj_result else: log.error('Something went wrong while trying to create a new event presentation.') # event_presentation_id = create_event_presentation_obj_result break log.debug(create_event_presentation_obj_result) # log.error('Hit continue to next loop point') # continue # log.error('Hit break point') # break log.info('# PART 3: Process the presenter data') presenter_data = {} presenter_data['for_type'] = 'event_presentation' presenter_data['for_id'] = event_presentation_id presenter_data['event_session_id'] = event_session_id presenter_data['event_presentation_id'] = event_presentation_id presenter_data['code'] = record.get('presenter_code') presenter_data['given_name'] = record.get('presenter_given_name') presenter_data['family_name'] = record.get('presenter_family_name') presenter_data['sort'] = record.get('presenter_code') # Using the code as the sort as well presenter_data['enable'] = True presenter_data['hide'] = False log.debug(presenter_data) data = {} data['event_presentation_id'] = event_presentation_id data['code'] = presenter_data['code'] sql = f""" SELECT `event_presenter`.id AS 'event_presenter_id', `event_presenter`.id_random AS 'event_presenter_id_random' FROM `event_presenter` AS `event_presenter` WHERE event_presenter.event_presentation_id = :event_presentation_id AND event_presenter.code = :code LIMIT 1; """ log.debug(sql) if event_presenter_rec_li_result := sql_select(data=data, sql=sql, as_list=True): log.info('Got a list result') event_presenter_rec = event_presenter_rec_li_result[0] else: # [] or False log.info('No results or something went wrong') event_presenter_rec = None log.debug(event_presenter_rec_li_result) event_presenter_id = None if event_presenter_rec: log.info('Found a record. Updating...') event_presenter_id = event_presenter_rec.get('event_presenter_id') if update_event_presenter_obj_result := update_event_presenter_obj_v3( event_presenter_id = event_presenter_id, event_presenter_obj_exist = presenter_data, event_presentation_id = event_presentation_id, ): pass else: log.error('Something went wrong while trying to update an event presenter.') # event_presenter_id = update_event_presenter_obj_result break log.debug(update_event_presenter_obj_result) else: log.info('No record found. Inserting...') if create_event_presenter_obj_result := create_event_presenter_obj( event_presentation_id = event_presentation_id, event_presenter_obj_new = presenter_data ): event_presenter_id = create_event_presenter_obj_result else: log.error('Something went wrong while trying to create a new event presenter.') # event_presenter_id = create_event_presenter_obj_result break log.debug(create_event_presenter_obj_result) # log.error('Hit break point') # break return mk_resp(data=session_dict_li, response=commons.response)