Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on IDAA data imports

Browse Source
This commit is contained in:
Scott Idem
2021-10-04 18:41:13 -04:00
parent 7e9a8a493d
commit 6bf0831566
2 changed files with 45 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/db_sql.py
View File

@@ -449,7 +449,7 @@ def sql_select(
as_list: bool|None = False, as_list: bool|None = False,
max_count: int = 100000, max_count: int = 100000,
): ):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
if table_name and not (record_id or record_id_random or field_name or field_value or sql or data): if table_name and not (record_id or record_id_random or field_name or field_value or sql or data):

74
app/routers/importing.py
View File

@@ -24,15 +24,17 @@ from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter() router = APIRouter()
# First round: Working importing the Cvent Address Book export data # Based on the Cvent Address Book Contacts export data with new External IDs added as needed.
@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base) # The import file should only contain records with new External IDs.
async def importing_person_data_v2_external_id( # Updated 2021-10-04
@router.get('/update_w_external_id', response_model=Resp_Body_Base)
async def importing_update_w_external_id(
response: Response = Response, response: Response = Response,
): ):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
account_id = 99 account_id = 13
full_file_path = 'admin/temp/import_person_external_id.xlsx' full_file_path = 'admin/temp/import_person_external_id.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str})
@@ -64,15 +66,16 @@ async def importing_person_data_v2_external_id(
FROM `v_person` AS `person` FROM `v_person` AS `person`
WHERE person.account_id = :account_id WHERE person.account_id = :account_id
AND person.email = :email AND person.email = :email
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1; LIMIT 1;
""" """
if person_rec_result := sql_select(data=data, sql=sql): if person_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing person record # Pull out IDs and UPDATE existing person record
log.debug('Found one record')
person_rec = person_rec_result person_rec = person_rec_result
person_id = person_rec.get('person_id', None) person_id = person_rec.get('person_id', None)
person_id_random = person_rec.get('person_id_random', None)
full_name = person_rec.get('full_name', None)
log.info(f'Found one record. Person ID: {person_id_random} Person Full Name: {full_name}')
# contact_id = person_rec.get('contact_id', None) # contact_id = person_rec.get('contact_id', None)
# address_id = person_rec.get('address_id', None) # address_id = person_rec.get('address_id', None)
# user_id = person_rec.get('user_id', None) # user_id = person_rec.get('user_id', None)
@@ -82,6 +85,7 @@ async def importing_person_data_v2_external_id(
person_data = {} person_data = {}
person_data['id'] = person_id person_data['id'] = person_id
person_data['external_id'] = external_id person_data['external_id'] = external_id
person_data['external_import_id'] = email
if person_obj_up_result := sql_update(data=person_data, table_name='person'): if person_obj_up_result := sql_update(data=person_data, table_name='person'):
log.debug(person_obj_up_result) log.debug(person_obj_up_result)
@@ -102,9 +106,11 @@ async def importing_person_data_v2_external_id(
return mk_resp(data=person_data_li) return mk_resp(data=person_data_li)
# First round: Working importing the Cvent Address Book export data # Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function.
@router.post('/person_data_v2', response_model=Resp_Body_Base) # All records should have an External ID.
async def importing_person_data_v2( # Updated 2021-10-04
@router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base)
async def ins_up_person_contact_address_user_data(
response: Response = Response, response: Response = Response,
): ):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
@@ -125,8 +131,9 @@ async def importing_person_data_v2(
account_id = 99 account_id = 99
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
# full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
#df = df.fillna('') # replace NaN with '' #df = df.fillna('') # replace NaN with ''
# df = df.fillna(None) # df = df.fillna(None)
# df = df.fillna('', inplace=True) # df = df.fillna('', inplace=True)
@@ -150,7 +157,7 @@ async def importing_person_data_v2(
# membership_person_id = None # membership_person_id = None
person_data = {} person_data = {}
if record['external_import_id']: if record.get('external_import_id'):
person_data['external_import_id'] = record['external_import_id'] person_data['external_import_id'] = record['external_import_id']
else: else:
person_data['external_import_id'] = record['email'] person_data['external_import_id'] = record['email']
@@ -166,27 +173,27 @@ async def importing_person_data_v2(
person_data['family_name'] = record['family_name'] person_data['family_name'] = record['family_name']
else: else:
person_data['family_name'] = None person_data['family_name'] = None
if name_prefix := record.get('name_prefix', None): person_data['title_names'] = name_prefix person_data['title_names'] = record.get('name_prefix', None)
if name_suffix := record.get('name_suffix', None): person_data['designations'] = name_suffix person_data['designations'] = record.get('name_suffix', None)
if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']:
person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
elif person_data['given_name'] and person_data['family_name']: elif person_data['given_name'] and person_data['family_name']:
person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] full_name = person_data['given_name']+' '+person_data['family_name']
elif person_data['given_name']: elif person_data['given_name']:
person_data['full_name'] = person_data['family_name'] full_name = person_data['family_name']
elif record['informal_full_name']: elif record['informal_full_name']:
person_data['full_name'] = record['informal_full_name'] full_name = record['informal_full_name']
elif record['informal_name']: elif record['informal_name']:
person_data['full_name'] = record['informal_name'] full_name = record['informal_name']
else: else:
person_data['full_name'] = None full_name = record['email']
if record.get('informal_full_name', None): # if record.get('informal_full_name', None):
person_data['informal_full_name'] = record['informal_full_name'] # person_data['informal_full_name'] = record['informal_full_name']
else: # else:
person_data['informal_full_name'] = None # person_data['informal_full_name'] = None
person_data['last_first_name'] = record.get('last_first_name', None) # person_data['last_first_name'] = record.get('last_first_name', None)
if designation := record.get('designation', None): if designation := record.get('designation', None): # This field is mostly empty in IDAA data from Cvent
if person_data['designations']: if person_data['designations']:
person_data['designations'] = person_data['designations'] + ' ' + designation # professional designation person_data['designations'] = person_data['designations'] + ' ' + designation # professional designation
else: else:
@@ -198,7 +205,8 @@ async def importing_person_data_v2(
person_data['designations'] = designation # professional designation person_data['designations'] = designation # professional designation
if birth_date := record.get('date_of_birth', None): if birth_date := record.get('date_of_birth', None):
person_data['birth_date'] = birth_date
person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y')
if gender_name := record.get('gender_non-binary', None): if gender_name := record.get('gender_non-binary', None):
if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1 if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1
@@ -263,6 +271,12 @@ async def importing_person_data_v2(
address_id = person_rec.get('address_id', None) address_id = person_rec.get('address_id', None)
user_id = person_rec.get('user_id', None) user_id = person_rec.get('user_id', None)
person_data['id'] = person_id person_data['id'] = person_id
if 'created_on' in person_data:
person_data.pop('created_on') # I don't want to reset the created date?
if 'updated_on' in person_data:
person_data.pop('updated_on') # I don't want to reset the updated date?
if allow_update_person: if allow_update_person:
if person_obj_up_result := sql_update(data=person_data, table_name='person'): if person_obj_up_result := sql_update(data=person_data, table_name='person'):
log.debug(person_obj_up_result) log.debug(person_obj_up_result)
@@ -427,7 +441,7 @@ async def importing_person_data_v2(
# Process the user data # Process the user data
log.debug('Process the user data') log.debug('Process the user data')
user_data = {} user_data = {}
user_data['name'] = person_data['full_name'] user_data['name'] = full_name # person_data['full_name']
user_data['username'] = record['email'] user_data['username'] = record['email']
user_data['email'] = record['email'] user_data['email'] = record['email']
user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start
@@ -497,10 +511,10 @@ async def importing_person_data_v2(
person_data_min = {} person_data_min = {}
person_data_min['person_id'] = person_id person_data_min['person_id'] = person_id
person_data_min['full_name'] = person_data['full_name'] person_data_min['full_name'] = full_name # person_data['full_name']
person_data_min['email'] = contact_data['email'] person_data_min['email'] = contact_data['email']
person_data_li.append(person_data_min) person_data_li.append(person_data_min)
log.debug(f"Record processed: {person_id} {person_data['full_name']}") log.debug(f"Record processed: {person_id} {full_name}")
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
# break # break