Working on data importing. Mainly for IDAA membership.
This commit is contained in:
Scott Idem
@@ -24,6 +24,84 @@ from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# First round: Working importing the Cvent Address Book export data
|
||||
@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base)
|
||||
async def importing_person_data_v2_external_id(
|
||||
response: Response = Response,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
account_id = 99
|
||||
full_file_path = 'admin/temp/import_person_external_id.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str})
|
||||
log.debug(df)
|
||||
|
||||
df_dict = df.to_dict(orient='records')
|
||||
|
||||
person_data_li = []
|
||||
|
||||
for record in df_dict:
|
||||
person_id = None
|
||||
if external_id := record.get('external_id', None): pass
|
||||
elif external_id := record.get('External ID', None): pass
|
||||
else:
|
||||
log.info('No external ID was found.')
|
||||
continue
|
||||
|
||||
if email := record.get('email', None): pass
|
||||
elif email := record.get('Email Address', None): pass
|
||||
else:
|
||||
log.info('No email address was found.')
|
||||
continue
|
||||
|
||||
data = {}
|
||||
data['account_id'] = account_id
|
||||
data['email'] = email
|
||||
sql = f"""
|
||||
SELECT *
|
||||
FROM `v_person` AS `person`
|
||||
WHERE person.account_id = :account_id
|
||||
AND person.email = :email
|
||||
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
|
||||
if person_rec_result := sql_select(data=data, sql=sql):
|
||||
# Pull out IDs and UPDATE existing person record
|
||||
log.debug('Found one record')
|
||||
person_rec = person_rec_result
|
||||
person_id = person_rec.get('person_id', None)
|
||||
# contact_id = person_rec.get('contact_id', None)
|
||||
# address_id = person_rec.get('address_id', None)
|
||||
# user_id = person_rec.get('user_id', None)
|
||||
# external_id = person_rec.get('external_id', None)
|
||||
# external_import_id = person_rec.get('external_import_id', None)
|
||||
|
||||
person_data = {}
|
||||
person_data['id'] = person_id
|
||||
person_data['external_id'] = external_id
|
||||
|
||||
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
|
||||
log.debug(person_obj_up_result)
|
||||
else:
|
||||
log.warning(person_obj_up_result)
|
||||
continue # Something unexpected may have happened
|
||||
person_rec = person_rec_result
|
||||
else:
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
person_data_min = {}
|
||||
person_data_min['person_id'] = person_id
|
||||
person_data_min['full_name'] = person_rec.get('full_name', None)
|
||||
person_data_min['email'] = email
|
||||
person_data_li.append(person_data_min)
|
||||
log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}")
|
||||
|
||||
return mk_resp(data=person_data_li)
|
||||
|
||||
|
||||
# First round: Working importing the Cvent Address Book export data
|
||||
@router.post('/person_data_v2', response_model=Resp_Body_Base)
|
||||
async def importing_person_data_v2(
|
||||
@@ -48,7 +126,7 @@ async def importing_person_data_v2(
|
||||
account_id = 99
|
||||
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
#df = df.fillna('') # replace NaN with ''
|
||||
# df = df.fillna(None)
|
||||
# df = df.fillna('', inplace=True)
|
||||
@@ -72,11 +150,11 @@ async def importing_person_data_v2(
|
||||
# membership_person_id = None
|
||||
|
||||
person_data = {}
|
||||
if record['external_id']:
|
||||
person_data['external_id'] = record['external_id']
|
||||
if record['external_import_id']:
|
||||
person_data['external_import_id'] = record['external_import_id']
|
||||
else:
|
||||
person_data['external_id'] = record['email']
|
||||
external_id = person_data['external_id']
|
||||
person_data['external_import_id'] = record['email']
|
||||
external_import_id = person_data['external_import_id']
|
||||
|
||||
if nickname := record.get('nickname', None): person_data['informal_name'] = nickname
|
||||
if given_name := record.get('given_name', None): person_data['given_name'] = given_name
|
||||
@@ -160,12 +238,12 @@ async def importing_person_data_v2(
|
||||
# continue
|
||||
data = {}
|
||||
data['account_id'] = account_id
|
||||
data['external_id'] = external_id
|
||||
data['external_import_id'] = external_import_id
|
||||
sql = f"""
|
||||
SELECT *
|
||||
FROM `v_person` AS `person`
|
||||
WHERE person.account_id = :account_id
|
||||
AND person.external_id = :external_id
|
||||
AND person.external_import_id = :external_import_id
|
||||
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
@@ -447,7 +525,7 @@ async def importing_person_data_v2_membership(
|
||||
account_id = 99
|
||||
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
log.debug(df)
|
||||
|
||||
df_dict = df.to_dict(orient='records')
|
||||
@@ -466,23 +544,23 @@ async def importing_person_data_v2_membership(
|
||||
|
||||
person_data = {}
|
||||
person_data['account_id'] = account_id
|
||||
if record['external_id']:
|
||||
person_data['external_id'] = record['external_id']
|
||||
if record['external_import_id']:
|
||||
person_data['external_import_id'] = record['external_import_id']
|
||||
else:
|
||||
person_data['external_id'] = record['email']
|
||||
external_id = person_data['external_id']
|
||||
person_data['external_import_id'] = record['email']
|
||||
external_import_id = person_data['external_import_id']
|
||||
|
||||
if membership_type_name := record.get('membership_type', None): pass
|
||||
else: continue
|
||||
|
||||
data = {}
|
||||
data['account_id'] = account_id
|
||||
data['external_id'] = external_id
|
||||
data['external_import_id'] = external_import_id
|
||||
sql = f"""
|
||||
SELECT *
|
||||
FROM `v_person` AS `person`
|
||||
WHERE person.account_id = :account_id
|
||||
AND person.external_id = :external_id
|
||||
AND person.external_import_id = :external_import_id
|
||||
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
|
||||
LIMIT 1;
|
||||
"""
|
||||
@@ -661,7 +739,7 @@ async def importing_person_data(
|
||||
account_id = 99
|
||||
full_file_path = 'admin/temp/import_person_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
#df = df.fillna('') # replace NaN with ''
|
||||
# df = df.fillna(None)
|
||||
# df = df.fillna('', inplace=True)
|
||||
@@ -683,10 +761,10 @@ async def importing_person_data(
|
||||
user_id = None
|
||||
|
||||
person_data = {}
|
||||
if record['external_id']:
|
||||
person_data['external_id'] = record['external_id']
|
||||
if record['external_import_id']:
|
||||
person_data['external_import_id'] = record['external_import_id']
|
||||
else:
|
||||
person_data['external_id'] = record['email']
|
||||
person_data['external_import_id'] = record['email']
|
||||
# person_data['informal_name'] = record['informal_name']
|
||||
person_data['given_name'] = record['given_name']
|
||||
if record['middle_name']:
|
||||
@@ -732,14 +810,14 @@ async def importing_person_data(
|
||||
person_data['meta_json'] = json.dumps(meta_data, indent=4)
|
||||
|
||||
|
||||
# Look up by email address or external ID and INSERT or UPDATE new person record
|
||||
# Look up by email address or external import ID and INSERT or UPDATE new person record
|
||||
# INSERT or UPDATE a contact record and address record if needed
|
||||
# INSERT or UPDATE a user record if needed
|
||||
# Process the person data
|
||||
log.debug(person_data)
|
||||
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
|
||||
# continue
|
||||
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']):
|
||||
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']):
|
||||
if not isinstance(person_rec_li_result, list):
|
||||
# Pull out IDs and UPDATE existing person record
|
||||
log.debug('Found one record')
|
||||
@@ -982,7 +1060,7 @@ async def importing_cont_edu_cert_person_data(
|
||||
cont_edu_cert_id = 3
|
||||
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
log.debug(df)
|
||||
|
||||
df_dict = df.to_dict(orient='records')
|
||||
@@ -1114,7 +1192,7 @@ async def importing_cont_edu_cert_person_data_touch(
|
||||
cont_edu_cert_id = 3
|
||||
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
log.debug(df)
|
||||
|
||||
df_dict = df.to_dict(orient='records')
|
||||
|
||||
Reference in New Issue
Block a user