Working on IDAA data imports
This commit is contained in:
Scott Idem
@@ -449,7 +449,7 @@ def sql_select(
|
||||
as_list: bool|None = False,
|
||||
max_count: int = 100000,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if table_name and not (record_id or record_id_random or field_name or field_value or sql or data):
|
||||
|
||||
@@ -24,15 +24,17 @@ from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# First round: Working importing the Cvent Address Book export data
|
||||
@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base)
|
||||
async def importing_person_data_v2_external_id(
|
||||
# Based on the Cvent Address Book Contacts export data with new External IDs added as needed.
|
||||
# The import file should only contain records with new External IDs.
|
||||
# Updated 2021-10-04
|
||||
@router.get('/update_w_external_id', response_model=Resp_Body_Base)
|
||||
async def importing_update_w_external_id(
|
||||
response: Response = Response,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
account_id = 99
|
||||
account_id = 13
|
||||
full_file_path = 'admin/temp/import_person_external_id.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str})
|
||||
@@ -64,15 +66,16 @@ async def importing_person_data_v2_external_id(
|
||||
FROM `v_person` AS `person`
|
||||
WHERE person.account_id = :account_id
|
||||
AND person.email = :email
|
||||
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
|
||||
if person_rec_result := sql_select(data=data, sql=sql):
|
||||
# Pull out IDs and UPDATE existing person record
|
||||
log.debug('Found one record')
|
||||
person_rec = person_rec_result
|
||||
person_id = person_rec.get('person_id', None)
|
||||
person_id_random = person_rec.get('person_id_random', None)
|
||||
full_name = person_rec.get('full_name', None)
|
||||
log.info(f'Found one record. Person ID: {person_id_random} Person Full Name: {full_name}')
|
||||
# contact_id = person_rec.get('contact_id', None)
|
||||
# address_id = person_rec.get('address_id', None)
|
||||
# user_id = person_rec.get('user_id', None)
|
||||
@@ -82,6 +85,7 @@ async def importing_person_data_v2_external_id(
|
||||
person_data = {}
|
||||
person_data['id'] = person_id
|
||||
person_data['external_id'] = external_id
|
||||
person_data['external_import_id'] = email
|
||||
|
||||
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
|
||||
log.debug(person_obj_up_result)
|
||||
@@ -102,9 +106,11 @@ async def importing_person_data_v2_external_id(
|
||||
return mk_resp(data=person_data_li)
|
||||
|
||||
|
||||
# First round: Working importing the Cvent Address Book export data
|
||||
@router.post('/person_data_v2', response_model=Resp_Body_Base)
|
||||
async def importing_person_data_v2(
|
||||
# Based on the Cvent Address Book Contacts export data. New External IDs should already be added in Cvent and updated in Aether as well. Use importing_update_w_external_id() endpoint function.
|
||||
# All records should have an External ID.
|
||||
# Updated 2021-10-04
|
||||
@router.get('/ins_up_person_contact_address_user_data', response_model=Resp_Body_Base)
|
||||
async def ins_up_person_contact_address_user_data(
|
||||
response: Response = Response,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
@@ -125,8 +131,9 @@ async def importing_person_data_v2(
|
||||
|
||||
account_id = 99
|
||||
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
|
||||
# full_file_path = 'admin/data_files/import_person_contact_address_user_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
#df = df.fillna('') # replace NaN with ''
|
||||
# df = df.fillna(None)
|
||||
# df = df.fillna('', inplace=True)
|
||||
@@ -150,7 +157,7 @@ async def importing_person_data_v2(
|
||||
# membership_person_id = None
|
||||
|
||||
person_data = {}
|
||||
if record['external_import_id']:
|
||||
if record.get('external_import_id'):
|
||||
person_data['external_import_id'] = record['external_import_id']
|
||||
else:
|
||||
person_data['external_import_id'] = record['email']
|
||||
@@ -166,27 +173,27 @@ async def importing_person_data_v2(
|
||||
person_data['family_name'] = record['family_name']
|
||||
else:
|
||||
person_data['family_name'] = None
|
||||
if name_prefix := record.get('name_prefix', None): person_data['title_names'] = name_prefix
|
||||
if name_suffix := record.get('name_suffix', None): person_data['designations'] = name_suffix
|
||||
person_data['title_names'] = record.get('name_prefix', None)
|
||||
person_data['designations'] = record.get('name_suffix', None)
|
||||
if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']:
|
||||
person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
|
||||
full_name = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
|
||||
elif person_data['given_name'] and person_data['family_name']:
|
||||
person_data['full_name'] = person_data['given_name']+' '+person_data['family_name']
|
||||
full_name = person_data['given_name']+' '+person_data['family_name']
|
||||
elif person_data['given_name']:
|
||||
person_data['full_name'] = person_data['family_name']
|
||||
full_name = person_data['family_name']
|
||||
elif record['informal_full_name']:
|
||||
person_data['full_name'] = record['informal_full_name']
|
||||
full_name = record['informal_full_name']
|
||||
elif record['informal_name']:
|
||||
person_data['full_name'] = record['informal_name']
|
||||
full_name = record['informal_name']
|
||||
else:
|
||||
person_data['full_name'] = None
|
||||
if record.get('informal_full_name', None):
|
||||
person_data['informal_full_name'] = record['informal_full_name']
|
||||
else:
|
||||
person_data['informal_full_name'] = None
|
||||
person_data['last_first_name'] = record.get('last_first_name', None)
|
||||
full_name = record['email']
|
||||
# if record.get('informal_full_name', None):
|
||||
# person_data['informal_full_name'] = record['informal_full_name']
|
||||
# else:
|
||||
# person_data['informal_full_name'] = None
|
||||
# person_data['last_first_name'] = record.get('last_first_name', None)
|
||||
|
||||
if designation := record.get('designation', None):
|
||||
if designation := record.get('designation', None): # This field is mostly empty in IDAA data from Cvent
|
||||
if person_data['designations']:
|
||||
person_data['designations'] = person_data['designations'] + ' ' + designation # professional designation
|
||||
else:
|
||||
@@ -198,7 +205,8 @@ async def importing_person_data_v2(
|
||||
person_data['designations'] = designation # professional designation
|
||||
|
||||
if birth_date := record.get('date_of_birth', None):
|
||||
person_data['birth_date'] = birth_date
|
||||
|
||||
person_data['birth_date'] = datetime.datetime.strptime(birth_date, '%d-%b-%Y')
|
||||
|
||||
if gender_name := record.get('gender_non-binary', None):
|
||||
if gender_name == 'Prefer not to say': person_data['lu_gender_id'] = 1
|
||||
@@ -263,6 +271,12 @@ async def importing_person_data_v2(
|
||||
address_id = person_rec.get('address_id', None)
|
||||
user_id = person_rec.get('user_id', None)
|
||||
person_data['id'] = person_id
|
||||
|
||||
if 'created_on' in person_data:
|
||||
person_data.pop('created_on') # I don't want to reset the created date?
|
||||
if 'updated_on' in person_data:
|
||||
person_data.pop('updated_on') # I don't want to reset the updated date?
|
||||
|
||||
if allow_update_person:
|
||||
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
|
||||
log.debug(person_obj_up_result)
|
||||
@@ -427,7 +441,7 @@ async def importing_person_data_v2(
|
||||
# Process the user data
|
||||
log.debug('Process the user data')
|
||||
user_data = {}
|
||||
user_data['name'] = person_data['full_name']
|
||||
user_data['name'] = full_name # person_data['full_name']
|
||||
user_data['username'] = record['email']
|
||||
user_data['email'] = record['email']
|
||||
user_data['email_verified'] = contact_data.get('email_active', False) # Not perfect, but a good start
|
||||
@@ -497,10 +511,10 @@ async def importing_person_data_v2(
|
||||
|
||||
person_data_min = {}
|
||||
person_data_min['person_id'] = person_id
|
||||
person_data_min['full_name'] = person_data['full_name']
|
||||
person_data_min['full_name'] = full_name # person_data['full_name']
|
||||
person_data_min['email'] = contact_data['email']
|
||||
person_data_li.append(person_data_min)
|
||||
log.debug(f"Record processed: {person_id} {person_data['full_name']}")
|
||||
log.debug(f"Record processed: {person_id} {full_name}")
|
||||
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
|
||||
# break
|
||||
|
||||
|
||||
Reference in New Issue
Block a user