From d1c7a88c64bbbd6f5e4d93402848a95690333765 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Wed, 18 Aug 2021 18:25:22 -0400 Subject: [PATCH] Working on data importing. Mainly for IDAA membership. --- app/db_sql.py | 4 +- app/methods/event_person_methods.py | 1 + app/models/contact_models.py | 1 + app/models/person_models.py | 9 + app/routers/importing.py | 577 ++++++++++++++++++++++++++++ 5 files changed, 590 insertions(+), 2 deletions(-) diff --git a/app/db_sql.py b/app/db_sql.py index 3b62e07..b1fd9f6 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -62,7 +62,7 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None, INSERT INTO `{table_name}` ({fields_string}) VALUES ({values_string}); """ ) - print(sql_insert) + # print(sql_insert) log.debug(f""" INSERT INTO `{table_name}` ({fields_string}) VALUES ({values_string}); @@ -301,7 +301,7 @@ def sql_select( max_count: int = 100000 ): current_log_level = log.level - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if table_name and not (record_id or record_id_random or field_name or field_value or sql or data): diff --git a/app/methods/event_person_methods.py b/app/methods/event_person_methods.py index f94f57e..75973c9 100644 --- a/app/methods/event_person_methods.py +++ b/app/methods/event_person_methods.py @@ -78,6 +78,7 @@ def load_event_person_obj( if inc_event_badge: log.info('Need to include event badge data...') # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + event_badge_id = event_person_rec.get('event_badge_id', None) if event_badge_obj := load_event_badge_obj( event_badge_id = event_badge_id ): diff --git a/app/models/contact_models.py b/app/models/contact_models.py index 936863e..4a64a1a 100644 --- a/app/models/contact_models.py +++ b/app/models/contact_models.py @@ -46,6 +46,7 @@ class Contact_Base(BaseModel): email: Optional[str] email_active: Optional[bool] email_status: Optional[str] + cc_email: Optional[str] phone_mobile: Optional[str] phone_home: Optional[str] diff --git a/app/models/person_models.py b/app/models/person_models.py index 7781917..08c1cd5 100644 --- a/app/models/person_models.py +++ b/app/models/person_models.py @@ -55,11 +55,19 @@ class Person_Base(BaseModel): display_name: Optional[str] # Custom whatever they want for public display title: Optional[str] + designation: Optional[str] organization_name: Optional[str] tagline: Optional[str] + birth_date: Optional[datetime.datetime] + lu_gender_id: Optional[int] + lu_gender_name: Optional[str] + + email_allowed: Optional[bool] + paper_mail_allowed: Optional[bool] + notes: Optional[str] created_on: Optional[datetime.datetime] = None @@ -68,6 +76,7 @@ class Person_Base(BaseModel): # Convenience Data # This is only for convenience. Probably going to keep unless it causes a problem. email: Optional[str] + cc_email: Optional[str] # Maybe add timezone in the future? # Including JSON data diff --git a/app/routers/importing.py b/app/routers/importing.py index 1bf65ee..c506c75 100644 --- a/app/routers/importing.py +++ b/app/routers/importing.py @@ -24,6 +24,583 @@ from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() +# First round: Working importing the Cvent Address Book export data +@router.post('/person_data_v2', response_model=Resp_Body_Base) +async def importing_person_data_v2( + response: Response = Response, + ): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + allow_insert_person = True + allow_insert_contact = True + allow_insert_address = True + allow_insert_user = True + # allow_insert_membership = True + + allow_update_person = True + allow_update_contact = True + allow_update_address = True + allow_update_user = True + # allow_update_membership = True + # allow_update_person_new_user = True + + account_id = 99 + full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' + + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + #df = df.fillna('') # replace NaN with '' + # df = df.fillna(None) + # df = df.fillna('', inplace=True) + #return str(df.info()) + log.debug(df) + + df_dict = df.to_dict(orient='records') + # log.debug(df_dict) + + # return mk_resp(data=False, status_code=500, response=response) + + person_data_li = [] + # for i in df.index: + for record in df_dict: + person_new = None + person_id = None + contact_id = None + address_id = None + user_id = None + # person_profile_id = None + # membership_person_id = None + + person_data = {} + if record['external_id']: + person_data['external_id'] = record['external_id'] + else: + person_data['external_id'] = record['email'] + external_id = person_data['external_id'] + + if nickname := record.get('nickname', None): person_data['informal_name'] = nickname + if given_name := record.get('given_name', None): person_data['given_name'] = given_name + if record['middle_name']: + person_data['middle_name'] = record['middle_name'] + else: + person_data['middle_name'] = None + if record['family_name']: + person_data['family_name'] = record['family_name'] + else: + person_data['family_name'] = None + if name_prefix := record.get('name_prefix', None): person_data['prefix'] = name_prefix + if name_suffix := record.get('name_suffix', None): person_data['suffix'] = name_suffix + if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: + person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] + elif person_data['given_name'] and person_data['family_name']: + person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] + elif person_data['given_name']: + person_data['full_name'] = person_data['family_name'] + elif record['informal_full_name']: + person_data['full_name'] = record['informal_full_name'] + elif record['informal_name']: + person_data['full_name'] = record['informal_name'] + else: + person_data['full_name'] = None + if record.get('informal_full_name', None): + person_data['informal_full_name'] = record['informal_full_name'] + else: + person_data['informal_full_name'] = None + person_data['last_first_name'] = record.get('last_first_name', None) + + if designation := record.get('designation', None): + person_data['designation'] = designation # professional designation + elif designation := record.get('professional_designations', None): + person_data['designation'] = designation + + if birth_date := record.get('date_of_birth', None): + person_data['birth_date'] = birth_date + + if gender_name := record.get('gender_non-binary', None): + if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 + if gender_name == 'Male': person_data['lu_gender_id'] = 2 + if gender_name == 'Female': person_data['lu_gender_id'] = 3 + if gender_name == 'Non-binary/third gender': person_data['lu_gender_id'] = 4 + + if email_opt_out := person_data.get('email_opt_out', None): + if email_opt_out == 'Yes': person_data['email_allowed'] = False + if email_opt_out == 'No': person_data['email_allowed'] = True + + if paper_mail_opt_out := person_data.get('paper_mail_opt_out', None): + if paper_mail_opt_out == 'Yes': person_data['paper_mail_allowed'] = False + if paper_mail_opt_out == 'No' or paper_mail_opt_out == 0: person_data['paper_mail_allowed'] = True + + if created_on := record.get('created_on', None): person_data['created_on'] = created_on + if updated_on := record.get('updated_on', None): person_data['updated_on'] = updated_on + + other_data = {} + other_data['contact_type'] = record.get('contact_type', None) + other_data['membership_type'] = record.get('membership_type', None) + other_data['membership_join_date'] = record.get('membership_join_date', None) + other_data['membership_expiration_date'] = record.get('membership_expiration_date', None) + other_data['membership_last_renewal_date'] = record.get('membership_last_renewal_date', None) + other_data['convention_history'] = record.get('convention_history', None) + + meta_data = {} + meta_data['created_by_method'] = record.get('created_by_method', None) + meta_data['created_by_date'] = record.get('created_by_date', None) + meta_data['created_by_name'] = record.get('created_by_name', None) + meta_data['modified_by'] = record.get('modified_by', None) + + person_data['other_json'] = json.dumps(other_data, indent=4) + person_data['meta_json'] = json.dumps(meta_data, indent=4) + + + # Look up by email address or external ID and INSERT or UPDATE new person record + # INSERT or UPDATE a contact record and address record if needed + # INSERT or UPDATE a user record if needed + # Process the person data + log.debug(person_data) + # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') + # continue + data = {} + data['account_id'] = account_id + data['external_id'] = external_id + sql = f""" + SELECT * + FROM `v_person` AS `person` + WHERE person.account_id = :account_id + AND person.external_id = :external_id + ORDER BY `person`.created_on DESC, `person`.updated_on DESC + LIMIT 1; + """ + + if person_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing person record + log.debug('Found one record') + person_rec = person_rec_result + person_id = person_rec.get('person_id', None) + contact_id = person_rec.get('contact_id', None) + address_id = person_rec.get('address_id', None) + user_id = person_rec.get('user_id', None) + person_data['id'] = person_id + if allow_update_person: + if person_obj_up_result := sql_update(data=person_data, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + continue # Something unexpected may have happened + else: + log.warning('Found more than one record') + log.warning(person_rec_result) + # Do nothing + continue # Something unexpected may have happened + person_rec = person_rec_result + else: + # INSERT new record + log.debug('Found no records or something went wrong') + person_data['account_id'] = account_id + if allow_insert_person: + if person_obj_in_result := sql_insert(data=person_data, table_name='person'): + log.debug(person_obj_in_result) + person_id = person_obj_in_result # Should be an int + person_new = True # Need to UPDATE this record after the contact, address, and user data is processed + else: + log.warning(person_obj_in_result) + continue # Something unexpected may have happened + + # Process the contact data + log.debug('Process the contact data') + contact_data = {} + contact_data['email'] = record['email'] + + if record.get('email_status', None): + contact_data['email_status'] = record['email_status'] + if record('email_status', None) != 'Undeliverable': + contact_data['email_active'] = True + else: + contact_data['email_active'] = False + # else: + # contact_data['email_active'] = None + # pass + # if record('email_status', None): + # contact_data['email_status'] = record['email_status'] + # else: + # contact_data['email_status'] = None + + if cc_email := record.get('cc_email', None): contact_data['cc_email'] = cc_email + + if record['phone_mobile']: + contact_data['phone_mobile'] = record['phone_mobile'] + else: + contact_data['phone_mobile'] = None + if record['phone_home']: + contact_data['phone_home'] = record['phone_home'] + else: + contact_data['phone_home'] = None + + if record['phone_home_fax'] and record['phone_work_fax']: + contact_data['phone_fax'] = record['phone_home_fax'] + contact_data['phone_other'] = record['phone_work_fax'] + elif record['phone_home_fax']: + contact_data['phone_fax'] = record['phone_home_fax'] + elif record['phone_work_fax']: + contact_data['phone_fax'] = record['phone_work_fax'] + + if record['phone_work']: + contact_data['phone_office'] = record['phone_work'] + else: + contact_data['phone_office'] = None + + if facebook_url := record.get('facebook_url', None): contact_data['facebook_url'] = facebook_url + if linkedin_url := record.get('linkedin_url', None): contact_data['linkedin_url'] = linkedin_url + if twitter_url := record.get('twitter_url', None): contact_data['twitter_url'] = twitter_url + + log.debug(contact_data) + if contact_id: + # UPDATE existing contact record + log.info('UPDATE existing contact record') + contact_data['id'] = contact_id + if allow_update_contact: + if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): + log.debug(contact_obj_up_result) + else: + log.warning(contact_obj_up_result) + continue # Something unexpected may have happened + elif person_id: + # INSERT new contact record and link to person record + log.info('INSERT new contact record and link to person record') + contact_data['account_id'] = account_id + contact_data['for_type'] = 'person' + contact_data['for_id'] = person_id + if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): + log.debug(contact_obj_in_result) + contact_id = contact_obj_in_result # Should be an int + person_new = True # Need to UPDATE this record after the contact, address, and user data is processed + else: + log.debug(contact_obj_in_result) + continue # Something unexpected may have happened + + # Process the contact address data + log.debug('Process the contact address data') + address_data = {} + if record['home_address_line_1']: + address_data['line_1'] = record['home_address_line_1'] + else: + address_data['line_1'] = None + if record['home_address_line_2']: + address_data['line_2'] = record['home_address_line_2'] + else: + address_data['line_2'] = None + if record['home_address_line_3']: + address_data['line_3'] = record['home_address_line_3'] + else: + address_data['line_3'] = None + if record['home_address_city']: + address_data['city'] = record['home_address_city'] + else: + address_data['city'] = None + if record.get('home_address_country_code', None) and record.get('home_address_state_province_code', None): + address_data['country_subdivision_code'] = record['home_address_country_code']+'-'+record['home_address_state_province_code'] + else: + address_data['country_subdivision_code'] = None + if record.get('home_address_state_province_name', None): address_data['state_province'] = record.get('home_address_state_province_name', None) + + if record['home_address_postal_code']: + address_data['postal_code'] = record['home_address_postal_code'] + else: + address_data['postal_code'] = None + if record.get('home_address_country_code', None): + address_data['country_alpha_2_code'] = record.get('home_address_country_code', None) + else: + address_data['country_alpha_2_code'] = None + if record.get('home_address_country_name', None): address_data['country'] = record.get('home_address_country_name', None) + + log.debug(address_data) + if address_id: + # UPDATE existing address record + log.info('UPDATE existing address record') + address_data['id'] = address_id + if allow_update_address: + if address_obj_up_result := sql_update(data=address_data, table_name='address'): + log.debug(address_obj_up_result) + else: + log.warning(address_obj_up_result) + # continue # Something unexpected may have happened + elif contact_id: + # INSERT new address record and link to contact record + log.info('INSERT new address record and link to contact record') + address_data['account_id'] = account_id + address_data['for_type'] = 'contact' + address_data['for_id'] = contact_id + if address_obj_in_result := sql_insert(data=address_data, table_name='address'): + log.debug(address_obj_in_result) + address_id = address_obj_in_result # Should be an int + person_new = True # Need to UPDATE this record after the contact, address, and user data is processed + else: + log.debug(address_obj_in_result) + # break + continue # Something unexpected may have happened + else: + log.error('No address ID to update or contact ID to create an address linked to the contact.') + return False + + # Process the user data + log.debug('Process the user data') + user_data = {} + user_data['name'] = person_data['full_name'] + user_data['username'] = record['email'] + user_data['email'] = record['email'] + user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start + + user_data['enable'] = False + user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) + user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) + + user_data['super'] = False + user_data['manager'] = False + user_data['administrator'] = False + + user_data['public'] = False + user_data['verified'] = True + user_data['notes'] = 'Created by importing list' + + log.debug(user_data) + if user_id: + # UPDATE existing user record + log.info('UPDATE existing user record') + user_data['id'] = user_id + if allow_update_user: + if user_obj_up_result := sql_update(data=user_data, table_name='user'): + log.debug(user_obj_up_result) + else: + log.warning(user_obj_up_result) + continue # Something unexpected may have happened + elif person_id: + # INSERT new user record and link to person record + log.info('INSERT new user record and link to person record') + user_data['account_id'] = account_id + user_data['person_id'] = person_id + random_password_string = secrets.token_urlsafe(8) + user_data['password'] = secure_hash_string(string=random_password_string) + other_data = {} + other_data['temp_password'] = random_password_string + user_data['other_json'] = json.dumps(other_data, indent=4) + + if allow_insert_user: + if user_obj_in_result := sql_insert(data=user_data, table_name='user'): + log.debug(user_obj_in_result) + user_id = user_obj_in_result # Should be an int + person_new = True # Need to UPDATE this record after the contact, address, and user data is processed + else: + log.debug(user_obj_in_result) + # break + continue # Something unexpected may have happened + else: + log.error('No user ID to update or person ID to create a user linked to the person.') + return False + + if person_new: + log.debug('Updating person record one more time since this is a new person') + person_data_up = {} + person_data_up['id'] = person_id + person_data_up['user_id'] = user_id + # random_password_string + # Don't need to update with the new contact or address IDs that were just created. + + if allow_insert_user: # Because this only matters on user INSERT + if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + # break + continue # Something unexpected may have happened + + person_data_min = {} + person_data_min['person_id'] = person_id + person_data_min['full_name'] = person_data['full_name'] + person_data_min['email'] = contact_data['email'] + person_data_li.append(person_data_min) + log.debug(f"Record processed: {person_id} {person_data['full_name']}") + # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') + # break + + return mk_resp(data=person_data_li) + + +# Second round: Creating membership records from the Cvent Address Book export data +@router.post('/person_data_v2_membership', response_model=Resp_Body_Base) +async def importing_person_data_v2_membership( + response: Response = Response, + ): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + allow_insert_person = True + allow_insert_contact = True + allow_insert_address = True + allow_insert_user = True + # allow_insert_membership = True + + allow_update_person = True + allow_update_contact = True + allow_update_address = True + allow_update_user = True + # allow_update_membership = True + # allow_update_person_new_user = True + + account_id = 99 + full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' + + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + log.debug(df) + + df_dict = df.to_dict(orient='records') + # log.debug(df_dict) + + person_data_li = [] + for record in df_dict: + membership_new = None + person_id = None + contact_id = None + address_id = None + user_id = None + # person_profile_id = None + membership_person_id = None + membership_type_person_id = None + + person_data = {} + person_data['account_id'] = account_id + if record['external_id']: + person_data['external_id'] = record['external_id'] + else: + person_data['external_id'] = record['email'] + external_id = person_data['external_id'] + + if membership_type_name := record.get('membership_type', None): pass + else: continue + + data = {} + data['account_id'] = account_id + data['external_id'] = external_id + sql = f""" + SELECT * + FROM `v_person` AS `person` + WHERE person.account_id = :account_id + AND person.external_id = :external_id + ORDER BY `person`.created_on DESC, `person`.updated_on DESC + LIMIT 1; + """ + if person_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs of existing person record + log.debug('Found one record') + person_rec = person_rec_result + person_id = person_rec.get('person_id', None) + contact_id = person_rec.get('contact_id', None) + address_id = person_rec.get('address_id', None) + user_id = person_rec.get('user_id', None) + membership_person_id = person_rec.get('membership_person_id', None) + else: + continue + + membership_person_data = {} + membership_type_person_data = {} + + membership_person_data['lu_membership_person_status_id'] = None # expiration is > now + membership_type_person_data['lu_membership_type_person_status_id'] = None # expiration is > now + + membership_person_data['first_approved_on'] = record.get('membership_join_date', None) + membership_type_person_data['first_approved_on'] = record.get('membership_join_date', None) + membership_person_data['first_start_on'] = record.get('membership_join_date', None) + membership_type_person_data['first_start_on'] = record.get('membership_join_date', None) + + if membership_last_renewal_date := record.get('membership_last_renewal_date', None): + membership_person_data['start_on'] = membership_last_renewal_date + membership_type_person_data['start_on'] = membership_last_renewal_date + else: + membership_person_data['start_on'] = record.get('membership_join_date', None) + membership_type_person_data['start_on'] = record.get('membership_join_date', None) + membership_person_data['end_on'] = record.get('membership_expiration_date', None) + membership_type_person_data['end_on'] = record.get('membership_expiration_date', None) + membership_person_data['last_end_on'] = record.get('membership_expiration_date', None) + membership_type_person_data['last_end_on'] = record.get('membership_expiration_date', None) + + # membership_type_person_data['membership_person_id'] = membership_person_id + if membership_type_name == 'Al-Anon Member': + membership_type_person_data['membership_type_id'] = 6 + membership_type_person_data['product_id'] = 13 + membership_person_data['level'] = 1 + elif membership_type_name == 'Annual Contribution': # Unsure... making affililate + membership_type_person_data['membership_type_id'] = 8 + membership_type_person_data['product_id'] = 13 + membership_person_data['level'] = 3 + elif membership_type_name == 'Doctoral Qualifying Member': + membership_type_person_data['membership_type_id'] = 5 + membership_type_person_data['product_id'] = 4 + membership_person_data['level'] = 1 + elif membership_type_name == 'Student Member': + membership_type_person_data['membership_type_id'] = 7 + membership_type_person_data['product_id'] = 14 + membership_person_data['level'] = 1 + + if membership_person_id: # Update the membership records + # sql_update() # UPDATE membership_person + if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): + log.debug(membership_person_obj_up_result) + else: + log.warning(membership_person_obj_up_result) + continue # Something unexpected may have happened + + # sql_update() # UPDATE membership_type_person + if membership_type_person_obj_up_result := sql_update(data=membership_type_person_data, table_name='membership_type_person'): + log.debug(membership_type_person_obj_up_result) + else: + log.warning(membership_type_person_obj_up_result) + continue # Something unexpected may have happened + else: # Create new membership records + membership_person_data['account_id'] = account_id + membership_person_data['person_id'] = person_id + membership_person_data['user_id'] = user_id + + # sql_insert() # INSERT new membership_person + if membership_person_obj_in_result := sql_insert(data=membership_person_data, table_name='membership_person'): + log.debug(membership_person_obj_in_result) + membership_person_id = membership_person_obj_in_result # Should be an int + membership_person_new = True # Need to UPDATE this record after the membership type person data is processed + else: + log.warning(membership_person_obj_in_result) + continue # Something unexpected may have happened + + # sql_insert() # INSERT new membership_type_person link using membership_person_id + membership_type_person_data['membership_person_id'] = membership_person_id + if membership_type_person_obj_in_result := sql_insert(data=membership_type_person_data, table_name='membership_type_person'): + log.debug(membership_type_person_obj_in_result) + membership_type_person_id = membership_type_person_obj_in_result # Should be an int + membership_type_person_new = True # Need to UPDATE this record after the membership type person data is processed + else: + log.warning(membership_type_person_obj_in_result) + continue # Something unexpected may have happened + + membership_person_data['membership_type_person_id'] = membership_type_person_id + # sql_update() # UPDATE membership_person with new membership_type_person_id + if membership_person_obj_up_result := sql_update(data=membership_person_data, table_name='membership_person'): + log.debug(membership_person_obj_up_result) + else: + log.warning(membership_person_obj_up_result) + continue # Something unexpected may have happened + + person_data_min = {} + person_data_min['person_id'] = person_id + person_data_min['membership_person_id'] = membership_person_id + person_data_min['membership_type_person_id'] = membership_type_person_id + person_data_min['membership_type_name'] = membership_type_name + person_data_li.append(person_data_min) + log.debug(f"Record processed: {person_id} {person_data['full_name']}") + # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') + # break + + return mk_resp(data=person_data_li) + + + + + + @router.post('/person_data', response_model=Resp_Body_Base) async def importing_person_data( response: Response = Response,