import datetime, json, pytz, secrets, time import pandas, xlrd # qrcode from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Response, status from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union from app.lib_general import log, logging, secure_hash_string from app.config import settings from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template from app.methods.contact_methods import load_contact_obj, update_contact_obj from app.methods.person_methods import create_update_person_obj_v4b, get_person_rec_list, load_person_obj from app.methods.user_methods import load_user_obj from app.models.contact_models import Contact_Base from app.models.person_models import Person_Base from app.models.user_models import User_Base, User_New_Base, User_Out_Base from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() ext_id_update_list = [ 'ZE21VR86~Wolfe.aaron@gmail.com', 'VG79LN79~pbrewer5326@gmail.com', 'YE78WS38~docsaper@gmail.com', 'BF91DK17~roseanne.andrews@pinerest.org', 'EJ55WS41~tonyfalk@gmail.com', 'SH19PV12~Karameyers92@gmail.com', 'XG50WH30~skyost@gmail.com', 'UK95WU97~jkerbles@gmail.com', 'OM54NG97~bestavillo@msn.com', 'BU28AA70~christylroberts@gmail.com', 'JA47IN89~Accorbett@gmail.com', 'MC52QH32~ferfe1966@gmail.com', 'SB67KT77~Shavonsm22@gmail.com', 'WB82SX13~donleo8455@gmail.com', 'NQ12DP52~Jenny.glendrange@bvgsunglasses.com', 'LD32UM92~Brendankduffy@gmail.com', 'AH11RW36~drpleitz@gmail.com', 'AL28VU80~Simons.olga@gmail.com', 'NF15AW72~Kotchmd@me.com', 'AC59CK71~michaelalvarezdds@gmail.com', 'BL92EK73~sadpea2@GMAIL.COM', 'GY65GW13~Geoffreydpm@gmail.com', 'QQ93LI30~sarahrubin3733@gmail.com', 'VP13AJ80~meredith.ann.vieira@gmail.com', 'VB35RE20~leahod@gmail.com', 'OB99TO12~Tim_kavanaugh@yahoo.com', 'RR66CD90~m.capoc@gmail.com', 'BL10TA67~docjeffrey@aol.com', 'CB53IF82~wscurtice@gmail.com', 'WE42VE66~ray.glendrange@bvgsunglasses.com', 'TL33WC87~jpperlman@perlmancenter.com', 'UO29CZ53~Kdcmdog@me.com', 'CW83DK42~Colleenlenorebarnes@gmail.com', 'AZ83NT69~walk1936@bellsouth.net', 'HH85VO87~Bannon.9@gmail.com', 'FQ91EE42~timothyjmiett@gmail.com', 'XJ84QV29~wrightideas@comcast.net', 'HU15CG82~Pedagoga77@gmail.com', 'VE19IM26~agathaambrozy@gmail.com', 'BH24NM18~kmartin@martindentistry.net', 'DG72CH97~Activelifeortho2@gmail.com', 'WH29FY81~Louisearle@gmail.com', 'XD53QP68~Cwashi19@gmail.com', 'ND69HA28~mmersereau@cvent.com', 'YS47DQ81~steve@rutlen.net', 'CY53NG16~bbowersmed@gmail.com', 'CU79PO93~joewalkden@outlook.com', 'NE44OA53~markdrabinski@comcast.net', 'MU26HY85~ajl3212@gmail.com', 'NS22YW81~vmartinez956@gmail.com', ] # Based on the Cvent Address Book Contacts export data with new External IDs added as needed. # Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-04 @router.get('/update_w_external_id', response_model=Resp_Body_Base) async def importing_update_w_external_id( response: Response = Response, ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 13 full_file_path = 'admin/temp/import_person_external_id.csv' log.info(f'Starting import for Account ID {account_id} File Name: {full_file_path}') df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id':str, 'External ID':str, 'source_id':str, 'Source ID':str, 'email':str, 'Email Address':str}) log.debug(df) df_dict = df.to_dict(orient='records') person_data_li = [] for record in df_dict: person_id = None if external_id := record.get('external_id', None): pass elif external_id := record.get('External ID', None): pass else: log.info('No external ID was found.') continue if source_id := record.get('source_id', None): pass elif source_id := record.get('Source ID', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass elif email := record.get('Email Address', None): pass else: log.info('No email address was found.') continue log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') data = {} data['account_id'] = account_id data['email'] = email sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.email = :email AND (external_id IS NULL OR external_import_id IS NULL) LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record person_rec = person_rec_result person_id = person_rec.get('person_id', None) person_id_random = person_rec.get('person_id_random', None) full_name = person_rec.get('full_name', None) log.warning(f'Found one record. Person ID: {person_id_random} Person Full Name: {full_name}') # contact_id = person_rec.get('contact_id', None) # address_id = person_rec.get('address_id', None) # user_id = person_rec.get('user_id', None) # external_id = person_rec.get('external_id', None) # external_import_id = person_rec.get('external_import_id', None) person_data = {} person_data['id'] = person_id person_data['external_id'] = external_id if source_id: person_data['external_import_id'] = source_id else: person_data['external_import_id'] = email log.debug(person_data) if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.info(f'No record found that needs to be updated. Email: {email}') continue person_data_min = {} person_data_min['person_id'] = person_id person_data_min['full_name'] = person_rec.get('full_name', None) person_data_min['email'] = email person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}") return mk_resp(data=person_data_li, response=response) # Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function. # All records should have an External ID. # Updated 2021-10-04 @router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base) async def ins_up_person_contact_address_user_data( begin_at: int = 0, end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_insert_person = True allow_insert_contact = True allow_insert_address = True allow_insert_user = True # allow_insert_membership = True allow_update_person = True allow_update_contact = True allow_update_address = True allow_update_user = True # allow_update_membership = True # allow_update_person_new_user = True account_id = 13 full_file_path = 'admin/temp/import_person_contact_address_user_data.csv' # full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx' df = pandas.read_csv( full_file_path, na_filter=False, dtype={'external_id':str, 'External ID':str, 'source_id':str, 'Source ID':str, 'email':str, 'Email Address':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) df.rename(columns={ 'Source ID': 'source_id', 'External ID': 'external_id', 'Contact Type': 'contact_type', 'Email Address': 'email', 'Nickname': 'informal_name', 'Prefix': 'name_prefix', 'First Name': 'given_name', 'Middle Name': 'middle_name', 'Last Name': 'family_name', 'Suffix': 'name_suffix', 'Title': 'title_unknown', 'Company': 'company_name', # affiliations 'CC Email Address': 'cc_email', 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in 'Convention History': 'convention_history', 'Date of Birth': 'date_of_birth', 'Non-Binary Gender': 'gender_non-binary', 'Gender': 'gender', 'Mobile Phone': 'phone_mobile', 'Home Phone': 'phone_home', 'Home Fax': 'phone_home_fax', 'Home Address 1': 'home_address_line_1', 'Home Address 2': 'home_address_line_2', 'Home Address 3': 'home_address_line_3', 'Home City': 'home_address_city', 'Home State': 'home_address_state_province_code', 'Home State Name': 'home_address_state_province_name', 'Home Country': 'home_address_country_name', # home_address_country_code 'Home ZIP/Postal Code': 'home_address_postal_code', 'Primary Address': 'primary_address', 'Membership Type': 'membership_type', 'Membership Join Date': 'membership_type_join_date', 'Membership Last Renewal Date': 'membership_type_last_renewal_date', 'Membership Expiration Date': 'membership_type_expiration_date', 'Opted-Out': 'email_opt_out', 'Paper Mail Opted Out': 'paper_mail_opt_out', 'I am joining IDAA as a': 'joining_as', 'Practice Specialty': 'practice_specialty', 'Practice Status': 'practice_status', 'Designation': 'designation', 'Professional Designations': 'professional_designations', 'Qualifying Degree': 'qualifying_degree', 'Recovery Programs': 'recovery_program', 'Scholarship History': 'scholarship_history', 'Sobriety Date': 'sobriety_date', 'Sobriety Date - data from old system': 'sobriety_date_old_sys', 'Emergency Contact': 'emergency_contact', 'SO Name Prefix': 'so_name_prefix', 'SO First Name': 'so_name_given_name', 'SO Last Name': 'so_name_family_name', 'SO Professional Designation': 'so_professional_designation', 'Guest Home City': 'guest_home_address_city', 'Guest Home State/Province Code': 'guest_home_address_state_province_code', 'Work Phone': 'phone_work', 'Work Fax': 'phone_work_fax', 'Work Address 1': 'work_address_line_1', 'Work Address 2': 'work_address_line_2', 'Work Address 3': 'work_address_line_3', 'Work City': 'work_address_city', 'Work State': 'work_address_state_province_code', 'Work State Name': 'work_address_state_province_name', 'Work ZIP/Postal Code': 'work_address_postal_code', 'Work Country': 'work_address_country_code', 'Created By': 'created_by_name', 'Creation Date': 'created_by_date', }, inplace = True) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) loop_count = 0 person_data_li = [] # for i in df.index: for record in df_dict: log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break person_new = None person_id = None contact_id = None address_id = None user_id = None # person_profile_id = None # membership_person_id = None if external_id := record.get('external_id', None): pass else: log.warning('No external ID was found.') continue # This block should generally not be needed -Scott 2021-12-17 # if external_id in ext_id_update_list: # log.info('In list') # pass # else: # # log.info('Not in list') # continue if source_id := record.get('source_id', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass else: log.warning('No email address was found.') continue log.info(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') person_data = {} person_data['external_id'] = external_id if source_id: person_data['external_import_id'] = source_id else: person_data['external_import_id'] = email external_import_id = person_data['external_import_id'] log.debug(external_import_id) if record.get('informal_name'): person_data['informal_name'] = record.get('informal_name') if record.get('given_name'): person_data['given_name'] = record.get('given_name') if record.get('middle_name'): person_data['middle_name'] = record.get('middle_name') if record.get('family_name'): person_data['family_name'] = record.get('family_name') if record.get('name_prefix'): person_data['title_names'] = record.get('name_prefix') if record.get('name_suffix'): person_data['designations'] = record.get('name_suffix') if record.get('informal_name'): person_data['informal_full_name'] = record.get('informal_name') if record.get('informal_name'): person_data['informal_display_name'] = record.get('informal_name') if person_data.get('given_name') and person_data.get('middle_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] elif person_data.get('given_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['family_name'] elif person_data.get('given_name'): full_name = person_data['family_name'] elif record.get('informal_full_name'): full_name = record['informal_full_name'] elif record.get('informal_name'): full_name = record['informal_name'] else: full_name = record.get('email') # if record.get('informal_full_name', None): # person_data['informal_full_name'] = record['informal_full_name'] # else: # person_data['informal_full_name'] = None # person_data['last_first_name'] = record.get('last_first_name', None) if designation := record.get('designation', ''): # This field is mostly empty in IDAA data from Cvent if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip()[:125] # professional designation else: person_data['designations'] = str(designation) # professional designation elif designation := record.get('professional_designations', ''): if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip()[:125] # professional designation else: person_data['designations'] = str(designation)[:125] # professional designation if birth_date := record.get('date_of_birth'): person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y').date() if gender_name := record.get('gender_non-binary'): if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 if gender_name == 'Male': person_data['lu_gender_id'] = 2 if gender_name == 'Female': person_data['lu_gender_id'] = 3 if gender_name == 'Non-binary/third gender': person_data['lu_gender_id'] = 4 if email_opt_out := person_data.get('email_opt_out'): if email_opt_out == 'Yes': person_data['email_allowed'] = False if email_opt_out == 'No': person_data['email_allowed'] = True if paper_mail_opt_out := person_data.get('paper_mail_opt_out'): if paper_mail_opt_out == 'Yes': person_data['paper_mail_allowed'] = False if paper_mail_opt_out == 'No' or paper_mail_opt_out == 0: person_data['paper_mail_allowed'] = True person_data['enable'] = True if created_on := record.get('created_on'): person_data['created_on'] = datetime.datetime.strptime(created_on, '%d-%b-%Y') if updated_on := record.get('updated_on'): person_data['updated_on'] = datetime.datetime.strptime(updated_on, '%d-%b-%Y') other_data = {} other_data['contact_type'] = record.get('contact_type', None) other_data['membership_type'] = record.get('membership_type', None) other_data['membership_join_date'] = record.get('membership_join_date', None) other_data['membership_expiration_date'] = record.get('membership_expiration_date', None) other_data['membership_last_renewal_date'] = record.get('membership_last_renewal_date', None) other_data['convention_history'] = record.get('convention_history', None) meta_data = {} meta_data['created_by_method'] = record.get('created_by_method', None) meta_data['created_by_date'] = record.get('created_by_date', None) meta_data['created_by_name'] = record.get('created_by_name', None) meta_data['modified_by'] = record.get('modified_by', None) person_data['other_json'] = json.dumps(other_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4) # Look up by email address or external ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed # Process the person data log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue data = {} data['account_id'] = account_id data['external_id'] = external_id data['external_import_id'] = external_import_id # sql = f""" # SELECT * # FROM `v_person` AS `person` # WHERE person.account_id = :account_id # AND person.external_import_id = :external_import_id # LIMIT 1; # """ sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_id = :external_id LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record log.debug(person_rec_result) person_rec = person_rec_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id', None) address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) # import_new_person(person_id=person_id, contact_id=contact_id, address=address_id, user_id=user_id) person_data['id'] = person_id log.info(f'Found Person ID: {person_id} Contact ID: {contact_id} Address ID: {address_id} User ID: {user_id}') if 'created_on' in person_data: person_data.pop('created_on') # I don't want to reset the created date? if 'updated_on' in person_data: person_data.pop('updated_on') # I don't want to reset the updated date? # person_data.pop('other_json') # person_data.pop('meta_json') if allow_update_person: if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Person updating not allowed') # Do nothing continue else: # INSERT new record log.debug('Found no records or something went wrong. Creating a new record...') person_data['account_id'] = account_id if allow_insert_person: if person_obj_in_result := sql_insert(data=person_data, table_name='person'): log.debug(person_obj_in_result) person_id = person_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(person_obj_in_result) continue # Something unexpected may have happened else: log.warning('Person inserting not allowed') # Do nothing continue # Process the contact data log.info('Process the contact data') contact_data = {} contact_data['email'] = record['email'] if record.get('email_status', None): contact_data['email_status'] = record['email_status'] if record('email_status', None) != 'Undeliverable': contact_data['email_active'] = True else: contact_data['email_active'] = False # else: # contact_data['email_active'] = None # pass # if record('email_status', None): # contact_data['email_status'] = record['email_status'] # else: # contact_data['email_status'] = None if cc_email := record.get('cc_email', None): contact_data['cc_email'] = cc_email if record['phone_mobile']: contact_data['phone_mobile'] = record['phone_mobile'] else: contact_data['phone_mobile'] = None if record['phone_home']: contact_data['phone_home'] = record['phone_home'] else: contact_data['phone_home'] = None if record['phone_home_fax'] and record['phone_work_fax']: contact_data['phone_fax'] = record['phone_home_fax'] contact_data['phone_other'] = record['phone_work_fax'] elif record['phone_home_fax']: contact_data['phone_fax'] = record['phone_home_fax'] elif record['phone_work_fax']: contact_data['phone_fax'] = record['phone_work_fax'] if record['phone_work']: contact_data['phone_office'] = record['phone_work'] else: contact_data['phone_office'] = None if record.get('facebook_url'): contact_data['facebook_url'] = record.get('facebook_url') if record.get('linkedin_url'): contact_data['linkedin_url'] = record.get('linkedin_url') if record.get('twitter_url'): contact_data['twitter_url'] = record.get('twitter_url') contact_data['enable'] = True log.debug(contact_data) if contact_id: # UPDATE existing contact record log.info('UPDATE existing contact record') contact_data['id'] = contact_id if allow_update_contact: if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): log.debug(contact_obj_up_result) else: log.warning(contact_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new contact record and link to person record log.info('INSERT new contact record and link to person record') contact_data['account_id'] = account_id contact_data['for_type'] = 'person' contact_data['for_id'] = person_id if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): log.debug(contact_obj_in_result) contact_id = contact_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(contact_obj_in_result) continue # Something unexpected may have happened # Process the contact address data log.info('Process the contact address data') address_data = {} if record['home_address_line_1']: address_data['line_1'] = record['home_address_line_1'] else: address_data['line_1'] = None if record['home_address_line_2']: address_data['line_2'] = record['home_address_line_2'] else: address_data['line_2'] = None if record['home_address_line_3']: address_data['line_3'] = record['home_address_line_3'] else: address_data['line_3'] = None if record['home_address_city']: address_data['city'] = record['home_address_city'] else: address_data['city'] = None if record.get('home_address_country_code', None) and record.get('home_address_state_province_code', None): address_data['country_subdivision_code'] = record['home_address_country_code']+'-'+record['home_address_state_province_code'] else: address_data['country_subdivision_code'] = None if record.get('home_address_state_province_name', None): address_data['state_province'] = record.get('home_address_state_province_name', None) if record['home_address_postal_code']: address_data['postal_code'] = record['home_address_postal_code'] else: address_data['postal_code'] = None if record.get('home_address_country_code', None): address_data['country_alpha_2_code'] = record.get('home_address_country_code', None) else: address_data['country_alpha_2_code'] = None if record.get('home_address_country_name', None): address_data['country'] = record.get('home_address_country_name', None) address_data['enable'] = True log.debug(address_data) if address_id: # UPDATE existing address record log.info('UPDATE existing address record') address_data['id'] = address_id if allow_update_address: if address_obj_up_result := sql_update(data=address_data, table_name='address'): log.debug(address_obj_up_result) else: log.warning(address_obj_up_result) # continue # Something unexpected may have happened elif contact_id: # INSERT new address record and link to contact record log.info('INSERT new address record and link to contact record') address_data['account_id'] = account_id address_data['for_type'] = 'contact' address_data['for_id'] = contact_id if address_obj_in_result := sql_insert(data=address_data, table_name='address'): log.debug(address_obj_in_result) address_id = address_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(address_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No address ID to update or contact ID to create an address linked to the contact.') return False # Process the user data log.info('Process the user data') user_data = {} user_data['name'] = full_name # person_data['full_name'] user_data['username'] = record['email'] user_data['email'] = record['email'] user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start user_data['enable'] = True user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) user_data['super'] = False user_data['manager'] = False # user_data['administrator'] = False user_data['public'] = False user_data['verified'] = True user_data['notes'] = 'Created by importing list' log.debug(user_data) if user_id: log.info('UPDATE existing user record') user_data['id'] = user_id user_data.pop('enable') user_data.pop('email_verified') # user_data.pop('administrator') user_data.pop('notes') log.debug(user_data) if allow_update_user: if user_obj_up_result := sql_update(data=user_data, table_name='user'): log.debug(user_obj_up_result) else: log.warning(user_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new user record and link to person record log.info('INSERT new user record and link to person record') user_data['account_id'] = account_id user_data['person_id'] = person_id random_password_string = secrets.token_urlsafe(8) user_data['password'] = secure_hash_string(string=random_password_string) other_data = {} other_data['temp_password'] = random_password_string user_data['other_json'] = json.dumps(other_data, indent=4) if allow_insert_user: if user_obj_in_result := sql_insert(data=user_data, table_name='user'): log.debug(user_obj_in_result) user_id = user_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(user_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No user ID to update or person ID to create a user linked to the person.') return False if person_new: log.debug('Updating person record one more time since this is a new person') person_data_up = {} person_data_up['id'] = person_id person_data_up['user_id'] = user_id # random_password_string # Don't need to update with the new contact or address IDs that were just created. if allow_insert_user: # Because this only matters on user INSERT if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) # break continue # Something unexpected may have happened person_data_min = {} person_data_min['person_id'] = person_id person_data_min['full_name'] = full_name person_data_min['email'] = email person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {full_name}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) # Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function. Person contact address user data needs to be updated. Use ins_up_person_contact_address_user_data() endpoint function. # All records should have an External ID. # Updated 2021-10-04 @router.get('/ins_up_membership_person_data', response_model=Resp_Body_Base) async def ins_up_membership_person_data( begin_at: int = 0, end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) log.info('Starting membership person data insert update...') allow_insert_person = True allow_insert_contact = True allow_insert_address = True allow_insert_user = True # allow_insert_membership = True allow_update_person = True allow_update_contact = True allow_update_address = True allow_update_user = True # allow_update_membership = True # allow_update_person_new_user = True account_id = 13 full_file_path = 'admin/temp/import_person_contact_address_user_data.csv' df = pandas.read_csv( full_file_path, na_filter=False, dtype={'external_id':str, 'External ID':str, 'source_id':str, 'Source ID':str, 'email':str, 'Email Address':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df.rename(columns={ 'Source ID': 'source_id', 'External ID': 'external_id', 'Contact Type': 'contact_type', 'Email Address': 'email', 'Nickname': 'informal_name', 'Prefix': 'name_prefix', 'First Name': 'given_name', 'Middle Name': 'middle_name', 'Last Name': 'family_name', 'Suffix': 'name_suffix', 'Title': 'title_unknown', 'Company': 'company_name', # affiliations 'CC Email Address': 'cc_email', 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in 'Convention History': 'convention_history', 'Date of Birth': 'date_of_birth', 'Non-Binary Gender': 'gender_non-binary', 'Gender': 'gender', 'Mobile Phone': 'phone_mobile', 'Home Phone': 'phone_home', 'Home Fax': 'phone_home_fax', 'Home Address 1': 'home_address_line_1', 'Home Address 2': 'home_address_line_2', 'Home Address 3': 'home_address_line_3', 'Home City': 'home_address_city', 'Home State': 'home_address_state_province_code', 'Home State Name': 'home_address_state_province_name', 'Home Country': 'home_address_country_name', # home_address_country_code 'Home ZIP/Postal Code': 'home_address_postal_code', 'Primary Address': 'primary_address', 'Membership Type': 'membership_type', 'Membership Join Date': 'membership_type_join_date', 'Membership Last Renewal Date': 'membership_type_last_renewal_date', 'Membership Expiration Date': 'membership_type_expiration_date', 'Opted-Out': 'email_opt_out', 'Paper Mail Opted Out': 'paper_mail_opt_out', 'I am joining IDAA as a': 'joining_as', 'Practice Specialty': 'practice_specialty', 'Practice Status': 'practice_status', 'Designation': 'designation', 'Professional Designations': 'professional_designations', 'Qualifying Degree': 'qualifying_degree', 'Recovery Programs': 'recovery_program', 'Scholarship History': 'scholarship_history', 'Sobriety Date': 'sobriety_date', 'Sobriety Date - data from old system': 'sobriety_date_old_sys', 'Emergency Contact': 'emergency_contact', 'SO Name Prefix': 'so_name_prefix', 'SO First Name': 'so_name_given_name', 'SO Last Name': 'so_name_family_name', 'SO Professional Designation': 'so_professional_designation', 'Guest Home City': 'guest_home_address_city', 'Guest Home State/Province Code': 'guest_home_address_state_province_code', 'Work Phone': 'phone_work', 'Work Fax': 'phone_work_fax', 'Work Address 1': 'work_address_line_1', 'Work Address 2': 'work_address_line_2', 'Work Address 3': 'work_address_line_3', 'Work City': 'work_address_city', 'Work State': 'work_address_state_province_code', 'Work State Name': 'work_address_state_province_name', 'Work ZIP/Postal Code': 'work_address_postal_code', 'Work Country': 'work_address_country_code', 'Created By': 'created_by_name', 'Creation Date': 'created_by_date', }, inplace = True) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) loop_count = 0 person_data_li = [] for record in df_dict: log.info(f'Loop Count: {loop_count}') loop_count = loop_count + 1 if loop_count <= begin_at: continue if loop_count > end_at: break membership_new = None person_id = None contact_id = None address_id = None user_id = None # person_profile_id = None membership_person_id = None membership_type_person_id = None person_data = {} person_data['account_id'] = account_id if external_id := record.get('external_id', None): pass else: log.warning('No external ID was found.') continue # This block should generally not be needed -Scott 2021-12-17 # if external_id in ext_id_update_list: # log.info('In list') # pass # else: # # log.info('Not in list') # continue if source_id := record.get('source_id', None): pass else: log.debug('No source ID was found.') pass if email := record.get('email', None): pass else: log.info('No email address was found.') continue log.info(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') person_data = {} if source_id: person_data['external_import_id'] = source_id else: person_data['external_import_id'] = email external_import_id = person_data['external_import_id'] log.debug(external_import_id) if membership_type_name := record.get('membership_type', None): pass else: log.warning(f'No membership information. Person ID: {person_id}, External ID: {external_id}, Email: {email}') continue data = {} data['account_id'] = account_id data['external_id'] = external_id data['external_import_id'] = external_import_id sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_id = :external_id /*AND person.external_import_id = :external_import_id*/ LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs of existing person record log.debug('Found one record') person_rec = person_rec_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id', None) address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) membership_person_id = person_rec.get('membership_person_id', None) log.info(f'Found membership_person_id: {membership_person_id}') else: log.warning(f'A person was not found with External ID: {external_id}') continue membership_person_data = {} membership_type_person_data = {} # datetime.datetime.strptime(date_time_str, '%m/%d/%Y') membership_person_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') # membership_type_person_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_data['first_start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_type_person_data['first_start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') if membership_type_last_renewal_date := record.get('membership_type_last_renewal_date', None): membership_person_data['start_on'] = datetime.datetime.strptime(membership_type_last_renewal_date, '%m/%d/%Y') membership_type_person_data['start_on'] = datetime.datetime.strptime(membership_type_last_renewal_date, '%m/%d/%Y') else: membership_person_data['start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_type_person_data['start_on'] = datetime.datetime.strptime(record.get('membership_type_join_date', None), '%m/%d/%Y') membership_person_data['end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_type_person_data['end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_person_data['last_end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') membership_type_person_data['last_end_on'] = datetime.datetime.strptime(record.get('membership_type_expiration_date', None), '%m/%d/%Y') current_datetime = datetime.datetime.now() if membership_person_data['end_on'] >= current_datetime: membership_person_data['lu_membership_person_status_id'] = 5 # 5 = active; expiration is > now membership_type_person_data['lu_membership_type_status_id'] = 5 # 5 = active; expiration is > now else: membership_person_data['lu_membership_person_status_id'] = 7 # 7 = inactive; expiration is < now membership_type_person_data['lu_membership_type_status_id'] = 7 # 7 = inactive; expiration is > now # membership_type_person_data['membership_person_id'] = membership_person_id if membership_type_name == 'Al-Anon Member' or membership_type_name == 'Al-Anon Members': membership_type_person_data['membership_type_id'] = 6 membership_type_person_data['product_id'] = 13 membership_person_data['level'] = 1 membership_type_person_data['level'] = 1 elif membership_type_name == 'Annual Contribution' or membership_type_name == 'Annual Contributions': # Unsure... making affiliate membership_type_person_data['membership_type_id'] = 8 membership_type_person_data['product_id'] = 13 membership_person_data['level'] = 3 membership_type_person_data['level'] = 3 elif membership_type_name == 'Doctoral Qualifying Member' or membership_type_name == 'Doctoral Qualifying Members': membership_type_person_data['membership_type_id'] = 5 membership_type_person_data['product_id'] = 4 membership_person_data['level'] = 1 membership_type_person_data['level'] = 1 elif membership_type_name == 'Student Member' or membership_type_name == 'Student Members': membership_type_person_data['membership_type_id'] = 7 membership_type_person_data['product_id'] = 14 membership_person_data['level'] = 1 membership_type_person_data['level'] = 1 membership_person_data['enable'] = True membership_type_person_data['enable'] = True if membership_person_id: # Update the membership records membership_person_data['id'] = membership_person_id # sql_update() # UPDATE membership_person log.info('UPDATE membership_person...') if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_up_result) else: log.warning(membership_person_obj_up_result) # continue # Something unexpected may have happened # sql_update() # UPDATE membership_type_person log.info('UPDATE membership_type_person...') data = {} data['membership_person_id'] = membership_person_id sql = f""" SELECT * FROM `v_membership_person` AS `membership_person` WHERE membership_person.id = :membership_person_id LIMIT 1; """ if membership_person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs of existing person record log.debug('Found one record') membership_person_rec = membership_person_rec_result membership_type_person_id = membership_person_rec.get('membership_type_person_id', None) log.info(f'Found membership_type_person_id: {membership_type_person_id}') else: continue membership_type_person_data['id'] = membership_type_person_id if membership_type_person_obj_up_result := sql_update(data=membership_type_person_data, table_name='membership_type_person'): log.debug(membership_type_person_obj_up_result) else: log.warning(membership_type_person_obj_up_result) continue # Something unexpected may have happened else: # Create new membership records membership_person_data['account_id'] = account_id membership_person_data['person_id'] = person_id membership_person_data['user_id'] = user_id # sql_insert() # INSERT new membership_person log.info('INSERT new membership_person...') if membership_person_obj_in_result := sql_insert(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_in_result) membership_person_id = membership_person_obj_in_result # Should be an int membership_person_new = True # Need to UPDATE this record after the membership type person data is processed else: log.warning(membership_person_obj_in_result) # continue # Something unexpected may have happened # sql_insert() # INSERT new membership_type_person link using membership_person_id log.info('INSERT new membership_type_person link using membership_person_id...') membership_type_person_data['membership_person_id'] = membership_person_id if membership_type_person_obj_in_result := sql_insert(data=membership_type_person_data, table_name='membership_type_person'): log.debug(membership_type_person_obj_in_result) membership_type_person_id = membership_type_person_obj_in_result # Should be an int membership_type_person_new = True # Need to UPDATE this record after the membership type person data is processed else: log.warning(membership_type_person_obj_in_result) # continue # Something unexpected may have happened membership_person_data['membership_type_person_id'] = membership_type_person_id # sql_update() # UPDATE membership_person with new membership_type_person_id log.info('UPDATE membership_person with new membership_type_person_id...') membership_person_data['id'] = membership_person_id if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): log.debug(membership_person_obj_up_result) else: log.warning(membership_person_obj_up_result) continue # Something unexpected may have happened person_data_min = {} person_data_min['person_id'] = person_id person_data_min['membership_person_id'] = membership_person_id person_data_min['membership_type_person_id'] = membership_type_person_id person_data_min['membership_type_name'] = membership_type_name person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {record['email']} External ID: {external_id}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) @router.post('/person_data', response_model=Resp_Body_Base) async def importing_person_data( response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) allow_insert_person = True allow_insert_contact = False allow_insert_address = False allow_insert_user = True allow_update_person = False allow_update_contact = False allow_update_address = False allow_update_user = False # allow_update_person_new_user = True account_id = 99 full_file_path = 'admin/temp/import_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) person_data_li = [] # for i in df.index: for record in df_dict: person_new = None person_id = None contact_id = None address_id = None user_id = None person_data = {} if record['external_import_id']: person_data['external_import_id'] = record['external_import_id'] else: person_data['external_import_id'] = record['email'] # person_data['informal_name'] = record['informal_name'] person_data['given_name'] = record['given_name'] if record['middle_name']: person_data['middle_name'] = record['middle_name'] else: person_data['middle_name'] = None if record['family_name']: person_data['family_name'] = record['family_name'] else: person_data['family_name'] = None # person_data['designations'] = record['prefix'] person_data['designations'] = record['suffix'] if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] elif person_data['given_name'] and person_data['family_name']: person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] elif person_data['given_name']: person_data['full_name'] = person_data['family_name'] elif record['informal_full_name']: person_data['full_name'] = record['informal_full_name'] elif record['informal_name']: person_data['full_name'] = record['informal_name'] else: person_data['full_name'] = None if record['informal_full_name']: person_data['informal_full_name'] = record['informal_full_name'] else: person_data['informal_full_name'] = None person_data['last_first_name'] = record['last_first_name'] person_data['created_on'] = record['created_on'] person_data['updated_on'] = record['updated_on'] other_data = {} other_data['contact_type'] = record['contact_type'] # ???? meta_data = {} meta_data['created_by_method'] = record['created_by_method'] meta_data['created_by_name'] = record['created_by_name'] meta_data['modified_by'] = record['modified_by'] meta_data['created_by_method'] = record['created_by_method'] person_data['other_json'] = json.dumps(other_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4) # Look up by email address or external import ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed # Process the person data log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']): if not isinstance(person_rec_li_result, list): # Pull out IDs and UPDATE existing person record log.debug('Found one record') person_rec = person_rec_li_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id_new', None) # Using _new from view until old contact_id is removed from person table address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) person_data['id'] = person_id if allow_update_person: if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(person_rec_li_result) # Do nothing continue # Something unexpected may have happened person_rec_li = person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') person_data['account_id'] = account_id if allow_insert_person: if person_obj_in_result := sql_insert(data=person_data, table_name='person'): log.debug(person_obj_in_result) person_id = person_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(person_obj_in_result) continue # Something unexpected may have happened # Process the contact data log.debug('Process the contact data') contact_data = {} contact_data['email'] = record['email'] if record['email_status'] != 'Undeliverable': contact_data['email_active'] = True else: contact_data['email_active'] = False contact_data['email_status'] = record['email_status'] if record['email_status']: contact_data['email_status'] = record['email_status'] else: contact_data['email_status'] = None if record['phone_mobile']: contact_data['phone_mobile'] = record['phone_mobile'] else: contact_data['phone_mobile'] = None if record['phone_home']: contact_data['phone_home'] = record['phone_home'] else: contact_data['phone_home'] = None if record['phone_fax']: contact_data['phone_fax'] = record['phone_fax'] elif record['phone_fax'] and record['phone_work_fax']: contact_data['phone_fax'] = record['phone_fax'] contact_data['phone_other'] = record['phone_work_fax'] if record['phone_work']: contact_data['phone_office'] = record['phone_work'] else: contact_data['phone_office'] = None log.debug(contact_data) if contact_id: # UPDATE existing contact record log.info('UPDATE existing contact record') contact_data['id'] = contact_id if allow_update_contact: if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): log.debug(contact_obj_up_result) else: log.warning(contact_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new contact record and link to person record log.info('INSERT new contact record and link to person record') contact_data['account_id'] = account_id contact_data['for_type'] = 'person' contact_data['for_id'] = person_id if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): log.debug(contact_obj_in_result) contact_id = contact_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(contact_obj_in_result) continue # Something unexpected may have happened # Process the contact address data log.debug('Process the contact address data') address_data = {} if record['address_line_1']: address_data['line_1'] = record['address_line_1'] else: address_data['line_1'] = None if record['address_line_2']: address_data['line_2'] = record['address_line_2'] else: address_data['line_2'] = None if record['address_line_3']: address_data['line_3'] = record['address_line_3'] else: address_data['line_3'] = None if record['address_city']: address_data['city'] = record['address_city'] else: address_data['city'] = None if record['address_state_province_code']: address_data['country_subdivision_code'] = record['address_country_code']+'-'+record['address_state_province_code'] else: address_data['country_subdivision_code'] = None if record['address_postal_code']: address_data['postal_code'] = record['address_postal_code'] else: address_data['postal_code'] = None if record['address_country_code']: address_data['country_alpha_2_code'] = record['address_country_code'] else: address_data['country_alpha_2_code'] = None log.debug(address_data) if address_id: # UPDATE existing address record log.info('UPDATE existing address record') address_data['id'] = address_id if allow_update_address: if address_obj_up_result := sql_update(data=address_data, table_name='address'): log.debug(address_obj_up_result) else: log.warning(address_obj_up_result) # continue # Something unexpected may have happened elif contact_id: # INSERT new address record and link to contact record log.info('INSERT new address record and link to contact record') address_data['account_id'] = account_id address_data['for_type'] = 'contact' address_data['for_id'] = contact_id if address_obj_in_result := sql_insert(data=address_data, table_name='address'): log.debug(address_obj_in_result) address_id = address_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(address_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No address ID to update or contact ID to create an address linked to the contact.') return False # Process the user data log.debug('Process the user data') user_data = {} user_data['name'] = person_data['full_name'] user_data['username'] = record['email'] user_data['email'] = record['email'] user_data['email_verified'] = contact_data['email_active'] # Not perfect, but a good start user_data['enable'] = False user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) user_data['super'] = False user_data['manager'] = False user_data['administrator'] = False user_data['public'] = False user_data['verified'] = True user_data['notes'] = 'Created by importing list' log.debug(user_data) if user_id: # UPDATE existing user record log.info('UPDATE existing user record') user_data['id'] = user_id if allow_update_user: if user_obj_up_result := sql_update(data=user_data, table_name='user'): log.debug(user_obj_up_result) else: log.warning(user_obj_up_result) continue # Something unexpected may have happened elif person_id: # INSERT new user record and link to person record log.info('INSERT new user record and link to person record') user_data['account_id'] = account_id user_data['person_id'] = person_id random_password_string = secrets.token_urlsafe(8) user_data['password'] = secure_hash_string(string=random_password_string) other_data = {} other_data['temp_password'] = random_password_string user_data['other_json'] = json.dumps(other_data, indent=4) if allow_insert_user: if user_obj_in_result := sql_insert(data=user_data, table_name='user'): log.debug(user_obj_in_result) user_id = user_obj_in_result # Should be an int person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.debug(user_obj_in_result) # break continue # Something unexpected may have happened else: log.error('No user ID to update or person ID to create a user linked to the person.') return False if person_new: log.debug('Updating person record one more time since this is a new person') person_data_up = {} person_data_up['id'] = person_id person_data_up['user_id'] = user_id # random_password_string # Don't need to update with the new contact or address IDs that were just created. if allow_insert_user: # Because this only matters on user INSERT if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): log.debug(person_obj_up_result) else: log.warning(person_obj_up_result) # break continue # Something unexpected may have happened person_data_li.append(person_data) log.debug(f"Record processed: {person_id} {person_data['full_name']}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=person_data_li) @router.post('/cont_edu_cert_person_data', response_model=Resp_Body_Base) async def importing_cont_edu_cert_person_data( response: Response = Response, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 19 cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) cont_edu_cert_person_data_li = [] # for i in df.index: for record in df_dict: cont_edu_cert_person_new = None person_id = None user_id = None cont_edu_cert_person_id = None cont_edu_cert_person_data = {} cont_edu_cert_person_data['cont_edu_cert_id'] = cont_edu_cert_id cont_edu_cert_person_data['enable'] = True cont_edu_cert_person_data['email'] = record['email'] cont_edu_cert_person_data['given_name'] = record['given_name'] if record.get('middle_name', None): cont_edu_cert_person_data['middle_name'] = record['middle_name'] else: cont_edu_cert_person_data['middle_name'] = None if record['family_name']: cont_edu_cert_person_data['family_name'] = record['family_name'] else: cont_edu_cert_person_data['family_name'] = None if cont_edu_cert_person_data['given_name'] and cont_edu_cert_person_data['middle_name'] and cont_edu_cert_person_data['family_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['given_name']+' '+cont_edu_cert_person_data['middle_name']+' '+cont_edu_cert_person_data['family_name'] elif cont_edu_cert_person_data['given_name'] and cont_edu_cert_person_data['family_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['given_name']+' '+cont_edu_cert_person_data['family_name'] elif cont_edu_cert_person_data['given_name']: cont_edu_cert_person_data['full_name'] = cont_edu_cert_person_data['family_name'] elif record.get('informal_full_name', None): cont_edu_cert_person_data['full_name'] = record['informal_full_name'] elif record['informal_name']: cont_edu_cert_person_data['full_name'] = record['informal_name'] else: cont_edu_cert_person_data['full_name'] = None if record.get('informal_full_name', None): cont_edu_cert_person_data['informal_full_name'] = record['informal_full_name'] else: cont_edu_cert_person_data['informal_full_name'] = None cont_edu_cert_person_data['last_first_name'] = record.get('last_first_name', None) cont_edu_cert_person_data['display_name'] = record.get('display_name', None) # cont_edu_cert_person_data['created_on'] = record['created_on'] # cont_edu_cert_person_data['updated_on'] = record['updated_on'] other_data = {} other_data['other_guest_of'] = record['other_guest_of'] other_data['other_guest_li'] = record['other_guest_li'] cont_edu_cert_person_data['other_json'] = json.dumps(other_data, indent=4) # Look up by email address and INSERT or UPDATE new cont_edu_cert_person record # Process the cont_edu_cert_person data log.debug(cont_edu_cert_person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if cont_edu_cert_person_rec_li_result := sql_select(table_name='v_cont_edu_cert_person', field_name='email', field_value=cont_edu_cert_person_data['email']): if not isinstance(cont_edu_cert_person_rec_li_result, list): # Pull out IDs and UPDATE existing cont_edu_cert_person record log.debug('Found one record') cont_edu_cert_person_rec = cont_edu_cert_person_rec_li_result # person_id = cont_edu_cert_person_rec.get('person_id', None) # user_id = cont_edu_cert_person_rec.get('user_id', None) cont_edu_cert_person_data['id'] = cont_edu_cert_person_id if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_up_result) else: log.warning(cont_edu_cert_person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(cont_edu_cert_person_rec_li_result) # Do nothing continue # Something unexpected may have happened cont_edu_cert_person_rec_li = cont_edu_cert_person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') cont_edu_cert_person_data['account_id'] = account_id if cont_edu_cert_person_obj_in_result := sql_insert(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_in_result) cont_edu_cert_person_id = cont_edu_cert_person_obj_in_result # Should be an int cont_edu_cert_person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(cont_edu_cert_person_obj_in_result) continue # Something unexpected may have happened # if cont_edu_cert_person_new: # log.debug('Updating person record one more time since this is a new person') # cont_edu_cert_person_data_up = {} # cont_edu_cert_person_data_up['id'] = person_id # cont_edu_cert_person_data_up['user_id'] = user_id # random_password_string # # Don't need to update with the new contact or address IDs that were just created. # if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data_up, table_name='cont_edu_cert_person'): # log.debug(cont_edu_cert_person_obj_up_result) # else: # log.warning(cont_edu_cert_person_obj_up_result) # # break # continue # Something unexpected may have happened cont_edu_cert_person_data_li.append(cont_edu_cert_person_data) log.debug(f"Record processed: {cont_edu_cert_person_id} {cont_edu_cert_person_data['full_name']}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break return mk_resp(data=cont_edu_cert_person_data_li) @router.post('/cont_edu_cert_person_data_touch', response_model=Resp_Body_Base) async def importing_cont_edu_cert_person_data_touch( response: Response = Response, ): log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) account_id = 19 cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) log.debug(df) df_dict = df.to_dict(orient='records') # log.debug(df_dict) # return mk_resp(data=False, status_code=500, response=response) cont_edu_cert_person_data_li = [] # for i in df.index: for record in df_dict: cont_edu_cert_person_new = None person_id = None user_id = None cont_edu_cert_person_id = None cont_edu_cert_person_data = {} # cont_edu_cert_person_data['cont_edu_cert_id'] = cont_edu_cert_id cont_edu_cert_person_data['enable'] = True cont_edu_cert_person_data['email'] = record['email'] other_data = {} other_data['last_event_date'] = '2021-08-01' other_data['other_guest_of'] = record['other_guest_of'] other_data['other_guest_li'] = record['other_guest_li'] cont_edu_cert_person_data['other_json'] = json.dumps(other_data, indent=4) # Look up by email address and INSERT or UPDATE new cont_edu_cert_person record # Process the cont_edu_cert_person data log.debug(cont_edu_cert_person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue if cont_edu_cert_person_rec_li_result := sql_select(table_name='v_cont_edu_cert_person', field_name='email', field_value=cont_edu_cert_person_data['email']): if not isinstance(cont_edu_cert_person_rec_li_result, list): # Pull out IDs and UPDATE existing cont_edu_cert_person record # log.debug('Found one record') cont_edu_cert_person_rec = cont_edu_cert_person_rec_li_result cont_edu_cert_person_id = cont_edu_cert_person_rec.get('cont_edu_cert_person_id', None) log.info(cont_edu_cert_person_id) # person_id = cont_edu_cert_person_rec.get('person_id', None) # user_id = cont_edu_cert_person_rec.get('user_id', None) cont_edu_cert_person_data['id'] = cont_edu_cert_person_id if cont_edu_cert_person_obj_up_result := sql_update(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): # log.debug(cont_edu_cert_person_obj_up_result) pass else: log.warning(cont_edu_cert_person_obj_up_result) continue # Something unexpected may have happened else: log.warning('Found more than one record') log.warning(cont_edu_cert_person_rec_li_result) # Do nothing continue # Something unexpected may have happened cont_edu_cert_person_rec_li = cont_edu_cert_person_rec_li_result else: # INSERT new record log.debug('Found no records or something went wrong') cont_edu_cert_person_data['account_id'] = account_id if cont_edu_cert_person_obj_in_result := sql_insert(data=cont_edu_cert_person_data, table_name='cont_edu_cert_person'): log.debug(cont_edu_cert_person_obj_in_result) cont_edu_cert_person_id = cont_edu_cert_person_obj_in_result # Should be an int cont_edu_cert_person_new = True # Need to UPDATE this record after the contact, address, and user data is processed else: log.warning(cont_edu_cert_person_obj_in_result) continue # Something unexpected may have happened cont_edu_cert_person_data_li.append(cont_edu_cert_person_data) log.debug(f"Record processed: {cont_edu_cert_person_id}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL return mk_resp(data=cont_edu_cert_person_data_li)