From 17a2722012b549e96586f265743d51d860b467de Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Fri, 8 Oct 2021 14:14:07 -0400 Subject: [PATCH] Work on IDAA importing --- app/routers/importing.py | 346 +++++++++++++++++++++++++++++++-------- 1 file changed, 279 insertions(+), 67 deletions(-) diff --git a/app/routers/importing.py b/app/routers/importing.py index bad512a..0adfb04 100644 --- a/app/routers/importing.py +++ b/app/routers/importing.py @@ -25,7 +25,7 @@ router = APIRouter() # Based on the Cvent Address Book Contacts export data with new External IDs added as needed. -# The import file should only contain records with new External IDs. +# Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. # Updated 2021-10-04 @router.get('/update_w_external_id', response_model=Resp_Body_Base) async def importing_update_w_external_id( @@ -58,7 +58,6 @@ async def importing_update_w_external_id( log.debug('No source ID was found.') pass - if email := record.get('email', None): pass elif email := record.get('Email Address', None): pass else: @@ -125,6 +124,8 @@ async def importing_update_w_external_id( # Updated 2021-10-04 @router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base) async def ins_up_person_contact_address_user_data( + begin_at: int = 0, + end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL @@ -144,14 +145,97 @@ async def ins_up_person_contact_address_user_data( # allow_update_person_new_user = True account_id = 13 - full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' + full_file_path = 'admin/temp/import_person_contact_address_user_data.csv' # full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_csv( + full_file_path, na_filter=False, + dtype={'external_id':str, + 'External ID':str, + 'source_id':str, + 'Source ID':str, + 'email':str, + 'Email Address':str, + 'phone_home':str, + 'phone_mobile':str, + 'city':str, + 'state_province':str, + 'address_postal_code':str, + 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) #return str(df.info()) + + df.rename(columns={ + 'Source ID': 'source_id', + 'External ID': 'external_id', + 'Contact Type': 'contact_type', + 'Email Address': 'email', + 'Nickname': 'informal_name', + 'Prefix': 'name_prefix', + 'First Name': 'given_name', + 'Middle Name': 'middle_name', + 'Last Name': 'family_name', + 'Suffix': 'name_suffix', + 'Title': 'title_unknown', + 'Company': 'company_name', # affiliations + 'CC Email Address': 'cc_email', + 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in + 'Convention History': 'convention_history', + 'Date of Birth': 'date_of_birth', + 'Non-Binary Gender': 'gender_non-binary', + 'Gender': 'gender', + 'Mobile Phone': 'phone_mobile', + 'Home Phone': 'phone_home', + 'Home Fax': 'phone_home_fax', + 'Home Address 1': 'home_address_line_1', + 'Home Address 2': 'home_address_line_2', + 'Home Address 3': 'home_address_line_3', + 'Home City': 'home_address_city', + 'Home State': 'home_address_state_province_code', + 'Home State Name': 'home_address_state_province_name', + 'Home Country': 'home_address_country_name', # home_address_country_code + 'Home ZIP/Postal Code': 'home_address_postal_code', + 'Primary Address': 'primary_address', + 'Membership Type': 'membership_type', + 'Membership Join Date': 'membership_type_join_date', + 'Membership Last Renewal Date': 'membership_type_last_renewal_date', + 'Membership Expiration Date': 'membership_type_expiration_date', + 'Opted-Out': 'email_opt_out', + 'Paper Mail Opted Out': 'paper_mail_opt_out', + 'I am joining IDAA as a': 'joining_as', + 'Practice Specialty': 'practice_specialty', + 'Practice Status': 'practice_status', + 'Designation': 'designation', + 'Professional Designations': 'professional_designations', + 'Qualifying Degree': 'qualifying_degree', + 'Recovery Programs': 'recovery_program', + 'Scholarship History': 'scholarship_history', + 'Sobriety Date': 'sobriety_date', + 'Sobriety Date - data from old system': 'sobriety_date_old_sys', + 'Emergency Contact': 'emergency_contact', + 'SO Name Prefix': 'so_name_prefix', + 'SO First Name': 'so_name_given_name', + 'SO Last Name': 'so_name_family_name', + 'SO Professional Designation': 'so_professional_designation', + 'Guest Home City': 'guest_home_address_city', + 'Guest Home State/Province Code': 'guest_home_address_state_province_code', + 'Work Phone': 'phone_work', + 'Work Fax': 'phone_work_fax', + 'Work Address 1': 'work_address_line_1', + 'Work Address 2': 'work_address_line_2', + 'Work Address 3': 'work_address_line_3', + 'Work City': 'work_address_city', + 'Work State': 'work_address_state_province_code', + 'Work State Name': 'work_address_state_province_name', + 'Work ZIP/Postal Code': 'work_address_postal_code', + 'Work Country': 'work_address_country_code', + 'Created By': 'created_by_name', + 'Creation Date': 'created_by_date', + }, + inplace = True) + log.debug(df) df_dict = df.to_dict(orient='records') @@ -159,9 +243,16 @@ async def ins_up_person_contact_address_user_data( # return mk_resp(data=False, status_code=500, response=response) + loop_count = 0 + person_data_li = [] # for i in df.index: for record in df_dict: + log.info(f'Loop Count: {loop_count}') + loop_count = loop_count + 1 + if loop_count <= begin_at: continue + if loop_count > end_at: break + person_new = None person_id = None contact_id = None @@ -170,38 +261,53 @@ async def ins_up_person_contact_address_user_data( # person_profile_id = None # membership_person_id = None - person_data = {} - if record.get('external_import_id'): - person_data['external_import_id'] = record['external_import_id'] + if external_id := record.get('external_id', None): pass else: - person_data['external_import_id'] = record['email'] + log.info('No external ID was found.') + continue + + if source_id := record.get('source_id', None): pass + else: + log.debug('No source ID was found.') + pass + + if email := record.get('email', None): pass + else: + log.info('No email address was found.') + continue + log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') + + person_data = {} + if source_id: + person_data['external_import_id'] = source_id + else: + person_data['external_import_id'] = email external_import_id = person_data['external_import_id'] log.info(external_import_id) - if nickname := record.get('nickname', None): person_data['informal_name'] = nickname - if given_name := record.get('given_name', None): person_data['given_name'] = given_name - if record['middle_name']: - person_data['middle_name'] = record['middle_name'] - else: - person_data['middle_name'] = None - if record['family_name']: - person_data['family_name'] = record['family_name'] - else: - person_data['family_name'] = None - person_data['title_names'] = record.get('name_prefix', None) - person_data['designations'] = record.get('name_suffix', None) - if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: + if record.get('informal_name'): person_data['informal_name'] = record.get('informal_name') + if record.get('given_name'): person_data['given_name'] = record.get('given_name') + if record.get('middle_name'): person_data['middle_name'] = record.get('middle_name') + if record.get('family_name'): person_data['family_name'] = record.get('family_name') + + if record.get('name_prefix'): person_data['title_names'] = record.get('name_prefix') + if record.get('name_suffix'): person_data['designations'] = record.get('name_suffix') + + if record.get('informal_name'): person_data['informal_full_name'] = record.get('informal_name') + if record.get('informal_name'): person_data['informal_display_name'] = record.get('informal_name') + + if person_data.get('given_name') and person_data.get('middle_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] - elif person_data['given_name'] and person_data['family_name']: + elif person_data.get('given_name') and person_data.get('family_name'): full_name = person_data['given_name']+' '+person_data['family_name'] - elif person_data['given_name']: + elif person_data.get('given_name'): full_name = person_data['family_name'] - elif record['informal_full_name']: + elif record.get('informal_full_name'): full_name = record['informal_full_name'] - elif record['informal_name']: + elif record.get('informal_name'): full_name = record['informal_name'] else: - full_name = record['email'] + full_name = record.get('email') # if record.get('informal_full_name', None): # person_data['informal_full_name'] = record['informal_full_name'] # else: @@ -209,36 +315,37 @@ async def ins_up_person_contact_address_user_data( # person_data['last_first_name'] = record.get('last_first_name', None) if designation := record.get('designation', ''): # This field is mostly empty in IDAA data from Cvent - if person_data['designations']: + if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip() # professional designation else: person_data['designations'] = str(designation) # professional designation elif designation := record.get('professional_designations', ''): - if person_data['designations']: + if person_data.get('designations'): person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip() # professional designation else: person_data['designations'] = str(designation) # professional designation - if birth_date := record.get('date_of_birth', None): - + if birth_date := record.get('date_of_birth'): person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y') - if gender_name := record.get('gender_non-binary', None): + if gender_name := record.get('gender_non-binary'): if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 if gender_name == 'Male': person_data['lu_gender_id'] = 2 if gender_name == 'Female': person_data['lu_gender_id'] = 3 if gender_name == 'Non-binary/third gender': person_data['lu_gender_id'] = 4 - if email_opt_out := person_data.get('email_opt_out', None): + if email_opt_out := person_data.get('email_opt_out'): if email_opt_out == 'Yes': person_data['email_allowed'] = False if email_opt_out == 'No': person_data['email_allowed'] = True - if paper_mail_opt_out := person_data.get('paper_mail_opt_out', None): + if paper_mail_opt_out := person_data.get('paper_mail_opt_out'): if paper_mail_opt_out == 'Yes': person_data['paper_mail_allowed'] = False if paper_mail_opt_out == 'No' or paper_mail_opt_out == 0: person_data['paper_mail_allowed'] = True - if created_on := record.get('created_on', None): person_data['created_on'] = created_on - if updated_on := record.get('updated_on', None): person_data['updated_on'] = updated_on + if created_on := record.get('created_on'): + person_data['created_on'] = datetime.datetime.strptime(created_on, '%d-%b-%Y') + if updated_on := record.get('updated_on'): + person_data['updated_on'] = datetime.datetime.strptime(updated_on, '%d-%b-%Y') other_data = {} other_data['contact_type'] = record.get('contact_type', None) @@ -257,7 +364,6 @@ async def ins_up_person_contact_address_user_data( person_data['other_json'] = json.dumps(other_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4) - # Look up by email address or external ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed @@ -273,13 +379,13 @@ async def ins_up_person_contact_address_user_data( FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_import_id = :external_import_id - ORDER BY `person`.created_on DESC, `person`.updated_on DESC LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record - log.debug('Found one record') + log.info('Found one record') + log.debug(person_rec_result) person_rec = person_rec_result person_id = person_rec.get('person_id', None) contact_id = person_rec.get('contact_id', None) @@ -299,11 +405,9 @@ async def ins_up_person_contact_address_user_data( log.warning(person_obj_up_result) continue # Something unexpected may have happened else: - log.warning('Found more than one record') - log.warning(person_rec_result) + log.warning('Person updating not allowed') # Do nothing - continue # Something unexpected may have happened - person_rec = person_rec_result + continue else: # INSERT new record log.debug('Found no records or something went wrong') @@ -316,6 +420,10 @@ async def ins_up_person_contact_address_user_data( else: log.warning(person_obj_in_result) continue # Something unexpected may have happened + else: + log.warning('Person inserting not allowed') + # Do nothing + continue # Process the contact data log.debug('Process the contact data') @@ -360,9 +468,9 @@ async def ins_up_person_contact_address_user_data( else: contact_data['phone_office'] = None - if facebook_url := record.get('facebook_url', None): contact_data['facebook_url'] = facebook_url - if linkedin_url := record.get('linkedin_url', None): contact_data['linkedin_url'] = linkedin_url - if twitter_url := record.get('twitter_url', None): contact_data['twitter_url'] = twitter_url + if record.get('facebook_url'): contact_data['facebook_url'] = record.get('facebook_url') + if record.get('linkedin_url'): contact_data['linkedin_url'] = record.get('linkedin_url') + if record.get('twitter_url'): contact_data['twitter_url'] = record.get('twitter_url') log.debug(contact_data) if contact_id: @@ -478,6 +586,10 @@ async def ins_up_person_contact_address_user_data( # UPDATE existing user record log.info('UPDATE existing user record') user_data['id'] = user_id + user_data.pop('enable') + user_data.pop('email_verified') + user_data.pop('administrator') + user_data.pop('notes') if allow_update_user: if user_obj_up_result := sql_update(data=user_data, table_name='user'): log.debug(user_obj_up_result) @@ -526,8 +638,8 @@ async def ins_up_person_contact_address_user_data( person_data_min = {} person_data_min['person_id'] = person_id - person_data_min['full_name'] = full_name # person_data['full_name'] - person_data_min['email'] = contact_data['email'] + person_data_min['full_name'] = full_name + person_data_min['email'] = email person_data_li.append(person_data_min) log.debug(f"Record processed: {person_id} {full_name}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') @@ -541,6 +653,8 @@ async def ins_up_person_contact_address_user_data( # Updated 2021-10-04 @router.get('/ins_up_membership_person_data', response_model=Resp_Body_Base) async def ins_up_membership_person_data( + begin_at: int = 0, + end_at: int = 20000, response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL @@ -564,14 +678,106 @@ async def ins_up_membership_person_data( account_id = 13 full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_csv( + full_file_path, na_filter=False, + dtype={'external_id':str, + 'External ID':str, + 'source_id':str, + 'Source ID':str, + 'email':str, + 'Email Address':str, + 'phone_home':str, + 'phone_mobile':str, + 'city':str, + 'state_province':str, + 'address_postal_code':str, + 'country':str}) + + df.rename(columns={ + 'Source ID': 'source_id', + 'External ID': 'external_id', + 'Contact Type': 'contact_type', + 'Email Address': 'email', + 'Nickname': 'informal_name', + 'Prefix': 'name_prefix', + 'First Name': 'given_name', + 'Middle Name': 'middle_name', + 'Last Name': 'family_name', + 'Suffix': 'name_suffix', + 'Title': 'title_unknown', + 'Company': 'company_name', # affiliations + 'CC Email Address': 'cc_email', + 'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in + 'Convention History': 'convention_history', + 'Date of Birth': 'date_of_birth', + 'Non-Binary Gender': 'gender_non-binary', + 'Gender': 'gender', + 'Mobile Phone': 'phone_mobile', + 'Home Phone': 'phone_home', + 'Home Fax': 'phone_home_fax', + 'Home Address 1': 'home_address_line_1', + 'Home Address 2': 'home_address_line_2', + 'Home Address 3': 'home_address_line_3', + 'Home City': 'home_address_city', + 'Home State': 'home_address_state_province_code', + 'Home State Name': 'home_address_state_province_name', + 'Home Country': 'home_address_country_name', # home_address_country_code + 'Home ZIP/Postal Code': 'home_address_postal_code', + 'Primary Address': 'primary_address', + 'Membership Type': 'membership_type', + 'Membership Join Date': 'membership_type_join_date', + 'Membership Last Renewal Date': 'membership_type_last_renewal_date', + 'Membership Expiration Date': 'membership_type_expiration_date', + 'Opted-Out': 'email_opt_out', + 'Paper Mail Opted Out': 'paper_mail_opt_out', + 'I am joining IDAA as a': 'joining_as', + 'Practice Specialty': 'practice_specialty', + 'Practice Status': 'practice_status', + 'Designation': 'designation', + 'Professional Designations': 'professional_designations', + 'Qualifying Degree': 'qualifying_degree', + 'Recovery Programs': 'recovery_program', + 'Scholarship History': 'scholarship_history', + 'Sobriety Date': 'sobriety_date', + 'Sobriety Date - data from old system': 'sobriety_date_old_sys', + 'Emergency Contact': 'emergency_contact', + 'SO Name Prefix': 'so_name_prefix', + 'SO First Name': 'so_name_given_name', + 'SO Last Name': 'so_name_family_name', + 'SO Professional Designation': 'so_professional_designation', + 'Guest Home City': 'guest_home_address_city', + 'Guest Home State/Province Code': 'guest_home_address_state_province_code', + 'Work Phone': 'phone_work', + 'Work Fax': 'phone_work_fax', + 'Work Address 1': 'work_address_line_1', + 'Work Address 2': 'work_address_line_2', + 'Work Address 3': 'work_address_line_3', + 'Work City': 'work_address_city', + 'Work State': 'work_address_state_province_code', + 'Work State Name': 'work_address_state_province_name', + 'Work ZIP/Postal Code': 'work_address_postal_code', + 'Work Country': 'work_address_country_code', + 'Created By': 'created_by_name', + 'Creation Date': 'created_by_date', + }, + inplace = True) + log.debug(df) df_dict = df.to_dict(orient='records') - log.debug(df_dict) + # log.debug(df_dict) + + # return mk_resp(data=False, status_code=500, response=response) + + loop_count = 0 person_data_li = [] for record in df_dict: + log.info(f'Loop Count: {loop_count}') + loop_count = loop_count + 1 + if loop_count <= begin_at: continue + if loop_count > end_at: break + membership_new = None person_id = None contact_id = None @@ -584,16 +790,29 @@ async def ins_up_membership_person_data( person_data = {} person_data['account_id'] = account_id - if record['external_id']: # This is now required for the membership portion 2021-10-04 - external_id = record['external_id'] - else: continue # external_id = None - log.info(f'External ID: {external_id}') - - if record['external_import_id']: - person_data['external_import_id'] = record['external_import_id'] + if external_id := record.get('external_id', None): pass else: - person_data['external_import_id'] = record['email'] + log.info('No external ID was found.') + continue + + if source_id := record.get('source_id', None): pass + else: + log.debug('No source ID was found.') + pass + + if email := record.get('email', None): pass + else: + log.info('No email address was found.') + continue + log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}') + + person_data = {} + if source_id: + person_data['external_import_id'] = source_id + else: + person_data['external_import_id'] = email external_import_id = person_data['external_import_id'] + log.info(external_import_id) if membership_type_name := record.get('membership_type', None): pass else: continue @@ -606,17 +825,11 @@ async def ins_up_membership_person_data( SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id - AND person.external_id = :external_id + /*AND person.external_id = :external_id*/ + AND person.external_import_id = :external_import_id LIMIT 1; """ - # sql = f""" - # SELECT * - # FROM `v_person` AS `person` - # WHERE person.account_id = :account_id - # AND person.external_import_id = :external_import_id - # ORDER BY `person`.created_on DESC, `person`.updated_on DESC - # LIMIT 1; - # """ + if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs of existing person record log.debug('Found one record') @@ -633,7 +846,6 @@ async def ins_up_membership_person_data( membership_person_data = {} membership_type_person_data = {} - # datetime.datetime.strptime(date_time_str, '%m/%d/%Y') membership_person_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_join_date', None), '%m/%d/%Y')