diff --git a/app/db_sql.py b/app/db_sql.py index 2f003da..5a7c93d 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -449,7 +449,7 @@ def sql_select( as_list: bool|None = False, max_count: int = 100000, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if table_name and not (record_id or record_id_random or field_name or field_value or sql or data): diff --git a/app/routers/importing.py b/app/routers/importing.py index 68bcb37..d0a2893 100644 --- a/app/routers/importing.py +++ b/app/routers/importing.py @@ -24,15 +24,17 @@ from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() -# First round: Working importing the Cvent Address Book export data -@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base) -async def importing_person_data_v2_external_id( +# Based on the Cvent Address Book Contacts export data with new External IDs added as needed. +# The import file should only contain records with new External IDs. +# Updated 2021-10-04 +@router.get('/update_w_external_id', response_model=Resp_Body_Base) +async def importing_update_w_external_id( response: Response = Response, ): - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - account_id = 99 + account_id = 13 full_file_path = 'admin/temp/import_person_external_id.xlsx' df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str}) @@ -64,15 +66,16 @@ async def importing_person_data_v2_external_id( FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.email = :email - ORDER BY `person`.created_on DESC, `person`.updated_on DESC LIMIT 1; """ if person_rec_result := sql_select(data=data, sql=sql): # Pull out IDs and UPDATE existing person record - log.debug('Found one record') person_rec = person_rec_result person_id = person_rec.get('person_id', None) + person_id_random = person_rec.get('person_id_random', None) + full_name = person_rec.get('full_name', None) + log.info(f'Found one record. Person ID: {person_id_random} Person Full Name: {full_name}') # contact_id = person_rec.get('contact_id', None) # address_id = person_rec.get('address_id', None) # user_id = person_rec.get('user_id', None) @@ -82,6 +85,7 @@ async def importing_person_data_v2_external_id( person_data = {} person_data['id'] = person_id person_data['external_id'] = external_id + person_data['external_import_id'] = email if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) @@ -102,9 +106,11 @@ async def importing_person_data_v2_external_id( return mk_resp(data=person_data_li) -# First round: Working importing the Cvent Address Book export data -@router.post('/person_data_v2', response_model=Resp_Body_Base) -async def importing_person_data_v2( +# Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function. +# All records should have an External ID. +# Updated 2021-10-04 +@router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base) +async def ins_up_person_contact_address_user_data( response: Response = Response, ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL @@ -125,8 +131,9 @@ async def importing_person_data_v2( account_id = 99 full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' + # full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) @@ -150,7 +157,7 @@ async def importing_person_data_v2( # membership_person_id = None person_data = {} - if record['external_import_id']: + if record.get('external_import_id'): person_data['external_import_id'] = record['external_import_id'] else: person_data['external_import_id'] = record['email'] @@ -166,27 +173,27 @@ async def importing_person_data_v2( person_data['family_name'] = record['family_name'] else: person_data['family_name'] = None - if name_prefix := record.get('name_prefix', None): person_data['title_names'] = name_prefix - if name_suffix := record.get('name_suffix', None): person_data['designations'] = name_suffix + person_data['title_names'] = record.get('name_prefix', None) + person_data['designations'] = record.get('name_suffix', None) if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: - person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] + full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] elif person_data['given_name'] and person_data['family_name']: - person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] + full_name = person_data['given_name']+' '+person_data['family_name'] elif person_data['given_name']: - person_data['full_name'] = person_data['family_name'] + full_name = person_data['family_name'] elif record['informal_full_name']: - person_data['full_name'] = record['informal_full_name'] + full_name = record['informal_full_name'] elif record['informal_name']: - person_data['full_name'] = record['informal_name'] + full_name = record['informal_name'] else: - person_data['full_name'] = None - if record.get('informal_full_name', None): - person_data['informal_full_name'] = record['informal_full_name'] - else: - person_data['informal_full_name'] = None - person_data['last_first_name'] = record.get('last_first_name', None) + full_name = record['email'] + # if record.get('informal_full_name', None): + # person_data['informal_full_name'] = record['informal_full_name'] + # else: + # person_data['informal_full_name'] = None + # person_data['last_first_name'] = record.get('last_first_name', None) - if designation := record.get('designation', None): + if designation := record.get('designation', None): # This field is mostly empty in IDAA data from Cvent if person_data['designations']: person_data['designations'] = person_data['designations'] + ' ' + designation # professional designation else: @@ -198,7 +205,8 @@ async def importing_person_data_v2( person_data['designations'] = designation # professional designation if birth_date := record.get('date_of_birth', None): - person_data['birth_date'] = birth_date + + person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y') if gender_name := record.get('gender_non-binary', None): if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 @@ -263,6 +271,12 @@ async def importing_person_data_v2( address_id = person_rec.get('address_id', None) user_id = person_rec.get('user_id', None) person_data['id'] = person_id + + if 'created_on' in person_data: + person_data.pop('created_on') # I don't want to reset the created date? + if 'updated_on' in person_data: + person_data.pop('updated_on') # I don't want to reset the updated date? + if allow_update_person: if person_obj_up_result := sql_update(data=person_data, table_name='person'): log.debug(person_obj_up_result) @@ -427,7 +441,7 @@ async def importing_person_data_v2( # Process the user data log.debug('Process the user data') user_data = {} - user_data['name'] = person_data['full_name'] + user_data['name'] = full_name # person_data['full_name'] user_data['username'] = record['email'] user_data['email'] = record['email'] user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start @@ -497,10 +511,10 @@ async def importing_person_data_v2( person_data_min = {} person_data_min['person_id'] = person_id - person_data_min['full_name'] = person_data['full_name'] + person_data_min['full_name'] = full_name # person_data['full_name'] person_data_min['email'] = contact_data['email'] person_data_li.append(person_data_min) - log.debug(f"Record processed: {person_id} {person_data['full_name']}") + log.debug(f"Record processed: {person_id} {full_name}") # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # break