Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Work on IDAA importing

Browse Source
This commit is contained in:
Scott Idem
2021-10-08 14:14:07 -04:00
parent ceb16dad93
commit 17a2722012
1 changed files with 279 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

346
app/routers/importing.py
View File

@@ -25,7 +25,7 @@ router = APIRouter()
# Based on the Cvent Address Book Contacts export data with new External IDs added as needed.
# The import file should only contain records with new External IDs.
# Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed.
# Updated 2021-10-04
@router.get('/update_w_external_id', response_model=Resp_Body_Base)
async def importing_update_w_external_id(
@@ -58,7 +58,6 @@ async def importing_update_w_external_id(
log.debug('No source ID was found.')
pass
if email := record.get('email', None): pass
elif email := record.get('Email Address', None): pass
else:
@@ -125,6 +124,8 @@ async def importing_update_w_external_id(
# Updated 2021-10-04
@router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base)
async def ins_up_person_contact_address_user_data(
begin_at: int = 0,
end_at: int = 20000,
response: Response = Response,
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -144,14 +145,97 @@ async def ins_up_person_contact_address_user_data(
# allow_update_person_new_user = True
account_id = 13
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
full_file_path = 'admin/temp/import_person_contact_address_user_data.csv'
# full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_csv(
full_file_path, na_filter=False,
dtype={'external_id':str,
'External ID':str,
'source_id':str,
'Source ID':str,
'email':str,
'Email Address':str,
'phone_home':str,
'phone_mobile':str,
'city':str,
'state_province':str,
'address_postal_code':str,
'country':str})
#df = df.fillna('') # replace NaN with ''
# df = df.fillna(None)
# df = df.fillna('', inplace=True)
#return str(df.info())
df.rename(columns={
'Source ID': 'source_id',
'External ID': 'external_id',
'Contact Type': 'contact_type',
'Email Address': 'email',
'Nickname': 'informal_name',
'Prefix': 'name_prefix',
'First Name': 'given_name',
'Middle Name': 'middle_name',
'Last Name': 'family_name',
'Suffix': 'name_suffix',
'Title': 'title_unknown',
'Company': 'company_name', # affiliations
'CC Email Address': 'cc_email',
'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in
'Convention History': 'convention_history',
'Date of Birth': 'date_of_birth',
'Non-Binary Gender': 'gender_non-binary',
'Gender': 'gender',
'Mobile Phone': 'phone_mobile',
'Home Phone': 'phone_home',
'Home Fax': 'phone_home_fax',
'Home Address 1': 'home_address_line_1',
'Home Address 2': 'home_address_line_2',
'Home Address 3': 'home_address_line_3',
'Home City': 'home_address_city',
'Home State': 'home_address_state_province_code',
'Home State Name': 'home_address_state_province_name',
'Home Country': 'home_address_country_name', # home_address_country_code
'Home ZIP/Postal Code': 'home_address_postal_code',
'Primary Address': 'primary_address',
'Membership Type': 'membership_type',
'Membership Join Date': 'membership_type_join_date',
'Membership Last Renewal Date': 'membership_type_last_renewal_date',
'Membership Expiration Date': 'membership_type_expiration_date',
'Opted-Out': 'email_opt_out',
'Paper Mail Opted Out': 'paper_mail_opt_out',
'I am joining IDAA as a': 'joining_as',
'Practice Specialty': 'practice_specialty',
'Practice Status': 'practice_status',
'Designation': 'designation',
'Professional Designations': 'professional_designations',
'Qualifying Degree': 'qualifying_degree',
'Recovery Programs': 'recovery_program',
'Scholarship History': 'scholarship_history',
'Sobriety Date': 'sobriety_date',
'Sobriety Date - data from old system': 'sobriety_date_old_sys',
'Emergency Contact': 'emergency_contact',
'SO Name Prefix': 'so_name_prefix',
'SO First Name': 'so_name_given_name',
'SO Last Name': 'so_name_family_name',
'SO Professional Designation': 'so_professional_designation',
'Guest Home City': 'guest_home_address_city',
'Guest Home State/Province Code': 'guest_home_address_state_province_code',
'Work Phone': 'phone_work',
'Work Fax': 'phone_work_fax',
'Work Address 1': 'work_address_line_1',
'Work Address 2': 'work_address_line_2',
'Work Address 3': 'work_address_line_3',
'Work City': 'work_address_city',
'Work State': 'work_address_state_province_code',
'Work State Name': 'work_address_state_province_name',
'Work ZIP/Postal Code': 'work_address_postal_code',
'Work Country': 'work_address_country_code',
'Created By': 'created_by_name',
'Creation Date': 'created_by_date',
},
inplace = True)
log.debug(df)
df_dict = df.to_dict(orient='records')
@@ -159,9 +243,16 @@ async def ins_up_person_contact_address_user_data(
# return mk_resp(data=False, status_code=500, response=response)
loop_count = 0
person_data_li = []
# for i in df.index:
for record in df_dict:
log.info(f'Loop Count: {loop_count}')
loop_count = loop_count + 1
if loop_count <= begin_at: continue
if loop_count > end_at: break
person_new = None
person_id = None
contact_id = None
@@ -170,38 +261,53 @@ async def ins_up_person_contact_address_user_data(
# person_profile_id = None
# membership_person_id = None
person_data = {}
if record.get('external_import_id'):
person_data['external_import_id'] = record['external_import_id']
if external_id := record.get('external_id', None): pass
else:
person_data['external_import_id'] = record['email']
log.info('No external ID was found.')
continue
if source_id := record.get('source_id', None): pass
else:
log.debug('No source ID was found.')
pass
if email := record.get('email', None): pass
else:
log.info('No email address was found.')
continue
log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}')
person_data = {}
if source_id:
person_data['external_import_id'] = source_id
else:
person_data['external_import_id'] = email
external_import_id = person_data['external_import_id']
log.info(external_import_id)
if nickname := record.get('nickname', None): person_data['informal_name'] = nickname
if given_name := record.get('given_name', None): person_data['given_name'] = given_name
if record['middle_name']:
person_data['middle_name'] = record['middle_name']
else:
person_data['middle_name'] = None
if record['family_name']:
person_data['family_name'] = record['family_name']
else:
person_data['family_name'] = None
person_data['title_names'] = record.get('name_prefix', None)
person_data['designations'] = record.get('name_suffix', None)
if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']:
if record.get('informal_name'): person_data['informal_name'] = record.get('informal_name')
if record.get('given_name'): person_data['given_name'] = record.get('given_name')
if record.get('middle_name'): person_data['middle_name'] = record.get('middle_name')
if record.get('family_name'): person_data['family_name'] = record.get('family_name')
if record.get('name_prefix'): person_data['title_names'] = record.get('name_prefix')
if record.get('name_suffix'): person_data['designations'] = record.get('name_suffix')
if record.get('informal_name'): person_data['informal_full_name'] = record.get('informal_name')
if record.get('informal_name'): person_data['informal_display_name'] = record.get('informal_name')
if person_data.get('given_name') and person_data.get('middle_name') and person_data.get('family_name'):
full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
elif person_data['given_name'] and person_data['family_name']:
elif person_data.get('given_name') and person_data.get('family_name'):
full_name = person_data['given_name']+' '+person_data['family_name']
elif person_data['given_name']:
elif person_data.get('given_name'):
full_name = person_data['family_name']
elif record['informal_full_name']:
elif record.get('informal_full_name'):
full_name = record['informal_full_name']
elif record['informal_name']:
elif record.get('informal_name'):
full_name = record['informal_name']
else:
full_name = record['email']
full_name = record.get('email')
# if record.get('informal_full_name', None):
# person_data['informal_full_name'] = record['informal_full_name']
# else:
@@ -209,36 +315,37 @@ async def ins_up_person_contact_address_user_data(
# person_data['last_first_name'] = record.get('last_first_name', None)
if designation := record.get('designation', ''): # This field is mostly empty in IDAA data from Cvent
if person_data['designations']:
if person_data.get('designations'):
person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip() # professional designation
else:
person_data['designations'] = str(designation) # professional designation
elif designation := record.get('professional_designations', ''):
if person_data['designations']:
if person_data.get('designations'):
person_data['designations'] = (str(person_data['designations']) + ' ' + str(designation)).strip() # professional designation
else:
person_data['designations'] = str(designation) # professional designation
if birth_date := record.get('date_of_birth', None):
if birth_date := record.get('date_of_birth'):
person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y')
if gender_name := record.get('gender_non-binary', None):
if gender_name := record.get('gender_non-binary'):
if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1
if gender_name == 'Male': person_data['lu_gender_id'] = 2
if gender_name == 'Female': person_data['lu_gender_id'] = 3
if gender_name == 'Non-binary/third gender': person_data['lu_gender_id'] = 4
if email_opt_out := person_data.get('email_opt_out', None):
if email_opt_out := person_data.get('email_opt_out'):
if email_opt_out == 'Yes': person_data['email_allowed'] = False
if email_opt_out == 'No': person_data['email_allowed'] = True
if paper_mail_opt_out := person_data.get('paper_mail_opt_out', None):
if paper_mail_opt_out := person_data.get('paper_mail_opt_out'):
if paper_mail_opt_out == 'Yes': person_data['paper_mail_allowed'] = False
if paper_mail_opt_out == 'No' or paper_mail_opt_out == 0: person_data['paper_mail_allowed'] = True
if created_on := record.get('created_on', None): person_data['created_on'] = created_on
if updated_on := record.get('updated_on', None): person_data['updated_on'] = updated_on
if created_on := record.get('created_on'):
person_data['created_on'] = datetime.datetime.strptime(created_on, '%d-%b-%Y')
if updated_on := record.get('updated_on'):
person_data['updated_on'] = datetime.datetime.strptime(updated_on, '%d-%b-%Y')
other_data = {}
other_data['contact_type'] = record.get('contact_type', None)
@@ -257,7 +364,6 @@ async def ins_up_person_contact_address_user_data(
person_data['other_json'] = json.dumps(other_data, indent=4)
person_data['meta_json'] = json.dumps(meta_data, indent=4)
# Look up by email address or external ID and INSERT or UPDATE new person record
# INSERT or UPDATE a contact record and address record if needed
# INSERT or UPDATE a user record if needed
@@ -273,13 +379,13 @@ async def ins_up_person_contact_address_user_data(
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.external_import_id = :external_import_id
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1;
"""
if person_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing person record
log.debug('Found one record')
log.info('Found one record')
log.debug(person_rec_result)
person_rec = person_rec_result
person_id = person_rec.get('person_id', None)
contact_id = person_rec.get('contact_id', None)
@@ -299,11 +405,9 @@ async def ins_up_person_contact_address_user_data(
log.warning(person_obj_up_result)
continue # Something unexpected may have happened
else:
log.warning('Found more than one record')
log.warning(person_rec_result)
log.warning('Person updating not allowed')
# Do nothing
continue # Something unexpected may have happened
person_rec = person_rec_result
continue
else:
# INSERT new record
log.debug('Found no records or something went wrong')
@@ -316,6 +420,10 @@ async def ins_up_person_contact_address_user_data(
else:
log.warning(person_obj_in_result)
continue # Something unexpected may have happened
else:
log.warning('Person inserting not allowed')
# Do nothing
continue
# Process the contact data
log.debug('Process the contact data')
@@ -360,9 +468,9 @@ async def ins_up_person_contact_address_user_data(
else:
contact_data['phone_office'] = None
if facebook_url := record.get('facebook_url', None): contact_data['facebook_url'] = facebook_url
if linkedin_url := record.get('linkedin_url', None): contact_data['linkedin_url'] = linkedin_url
if twitter_url := record.get('twitter_url', None): contact_data['twitter_url'] = twitter_url
if record.get('facebook_url'): contact_data['facebook_url'] = record.get('facebook_url')
if record.get('linkedin_url'): contact_data['linkedin_url'] = record.get('linkedin_url')
if record.get('twitter_url'): contact_data['twitter_url'] = record.get('twitter_url')
log.debug(contact_data)
if contact_id:
@@ -478,6 +586,10 @@ async def ins_up_person_contact_address_user_data(
# UPDATE existing user record
log.info('UPDATE existing user record')
user_data['id'] = user_id
user_data.pop('enable')
user_data.pop('email_verified')
user_data.pop('administrator')
user_data.pop('notes')
if allow_update_user:
if user_obj_up_result := sql_update(data=user_data, table_name='user'):
log.debug(user_obj_up_result)
@@ -526,8 +638,8 @@ async def ins_up_person_contact_address_user_data(
person_data_min = {}
person_data_min['person_id'] = person_id
person_data_min['full_name'] = full_name # person_data['full_name']
person_data_min['email'] = contact_data['email']
person_data_min['full_name'] = full_name
person_data_min['email'] = email
person_data_li.append(person_data_min)
log.debug(f"Record processed: {person_id} {full_name}")
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
@@ -541,6 +653,8 @@ async def ins_up_person_contact_address_user_data(
# Updated 2021-10-04
@router.get('/ins_up_membership_person_data', response_model=Resp_Body_Base)
async def ins_up_membership_person_data(
begin_at: int = 0,
end_at: int = 20000,
response: Response = Response,
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -564,14 +678,106 @@ async def ins_up_membership_person_data(
account_id = 13
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_csv(
full_file_path, na_filter=False,
dtype={'external_id':str,
'External ID':str,
'source_id':str,
'Source ID':str,
'email':str,
'Email Address':str,
'phone_home':str,
'phone_mobile':str,
'city':str,
'state_province':str,
'address_postal_code':str,
'country':str})
df.rename(columns={
'Source ID': 'source_id',
'External ID': 'external_id',
'Contact Type': 'contact_type',
'Email Address': 'email',
'Nickname': 'informal_name',
'Prefix': 'name_prefix',
'First Name': 'given_name',
'Middle Name': 'middle_name',
'Last Name': 'family_name',
'Suffix': 'name_suffix',
'Title': 'title_unknown',
'Company': 'company_name', # affiliations
'CC Email Address': 'cc_email',
'Confirmed Opted-In': 'opt-in_confirmed', # confirmed_opted-in
'Convention History': 'convention_history',
'Date of Birth': 'date_of_birth',
'Non-Binary Gender': 'gender_non-binary',
'Gender': 'gender',
'Mobile Phone': 'phone_mobile',
'Home Phone': 'phone_home',
'Home Fax': 'phone_home_fax',
'Home Address 1': 'home_address_line_1',
'Home Address 2': 'home_address_line_2',
'Home Address 3': 'home_address_line_3',
'Home City': 'home_address_city',
'Home State': 'home_address_state_province_code',
'Home State Name': 'home_address_state_province_name',
'Home Country': 'home_address_country_name', # home_address_country_code
'Home ZIP/Postal Code': 'home_address_postal_code',
'Primary Address': 'primary_address',
'Membership Type': 'membership_type',
'Membership Join Date': 'membership_type_join_date',
'Membership Last Renewal Date': 'membership_type_last_renewal_date',
'Membership Expiration Date': 'membership_type_expiration_date',
'Opted-Out': 'email_opt_out',
'Paper Mail Opted Out': 'paper_mail_opt_out',
'I am joining IDAA as a': 'joining_as',
'Practice Specialty': 'practice_specialty',
'Practice Status': 'practice_status',
'Designation': 'designation',
'Professional Designations': 'professional_designations',
'Qualifying Degree': 'qualifying_degree',
'Recovery Programs': 'recovery_program',
'Scholarship History': 'scholarship_history',
'Sobriety Date': 'sobriety_date',
'Sobriety Date - data from old system': 'sobriety_date_old_sys',
'Emergency Contact': 'emergency_contact',
'SO Name Prefix': 'so_name_prefix',
'SO First Name': 'so_name_given_name',
'SO Last Name': 'so_name_family_name',
'SO Professional Designation': 'so_professional_designation',
'Guest Home City': 'guest_home_address_city',
'Guest Home State/Province Code': 'guest_home_address_state_province_code',
'Work Phone': 'phone_work',
'Work Fax': 'phone_work_fax',
'Work Address 1': 'work_address_line_1',
'Work Address 2': 'work_address_line_2',
'Work Address 3': 'work_address_line_3',
'Work City': 'work_address_city',
'Work State': 'work_address_state_province_code',
'Work State Name': 'work_address_state_province_name',
'Work ZIP/Postal Code': 'work_address_postal_code',
'Work Country': 'work_address_country_code',
'Created By': 'created_by_name',
'Creation Date': 'created_by_date',
},
inplace = True)
log.debug(df)
df_dict = df.to_dict(orient='records')
log.debug(df_dict)
# log.debug(df_dict)
# return mk_resp(data=False, status_code=500, response=response)
loop_count = 0
person_data_li = []
for record in df_dict:
log.info(f'Loop Count: {loop_count}')
loop_count = loop_count + 1
if loop_count <= begin_at: continue
if loop_count > end_at: break
membership_new = None
person_id = None
contact_id = None
@@ -584,16 +790,29 @@ async def ins_up_membership_person_data(
person_data = {}
person_data['account_id'] = account_id
if record['external_id']: # This is now required for the membership portion 2021-10-04
external_id = record['external_id']
else: continue # external_id = None
log.info(f'External ID: {external_id}')
if record['external_import_id']:
person_data['external_import_id'] = record['external_import_id']
if external_id := record.get('external_id', None): pass
else:
person_data['external_import_id'] = record['email']
log.info('No external ID was found.')
continue
if source_id := record.get('source_id', None): pass
else:
log.debug('No source ID was found.')
pass
if email := record.get('email', None): pass
else:
log.info('No email address was found.')
continue
log.debug(f'External ID: {external_id}, Source ID {source_id}, Email: {email}')
person_data = {}
if source_id:
person_data['external_import_id'] = source_id
else:
person_data['external_import_id'] = email
external_import_id = person_data['external_import_id']
log.info(external_import_id)
if membership_type_name := record.get('membership_type', None): pass
else: continue
@@ -606,17 +825,11 @@ async def ins_up_membership_person_data(
SELECT *
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.external_id = :external_id
/*AND person.external_id = :external_id*/
AND person.external_import_id = :external_import_id
LIMIT 1;
"""
# sql = f"""
# SELECT *
# FROM `v_person` AS `person`
# WHERE person.account_id = :account_id
# AND person.external_import_id = :external_import_id
# ORDER BY `person`.created_on DESC, `person`.updated_on DESC
# LIMIT 1;
# """
if person_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs of existing person record
log.debug('Found one record')
@@ -633,7 +846,6 @@ async def ins_up_membership_person_data(
membership_person_data = {}
membership_type_person_data = {}
# datetime.datetime.strptime(date_time_str, '%m/%d/%Y')
membership_person_data['first_approved_on'] = datetime.datetime.strptime(record.get('membership_join_date', None), '%m/%d/%Y')