Working on data importing. Mainly for IDAA membership.

This commit is contained in:
Scott Idem
2021-08-19 15:10:59 -04:00
parent 02f719d206
commit 4eaeb152fe
6 changed files with 222 additions and 43 deletions

View File

@@ -46,7 +46,7 @@ def load_membership_person_obj(
inc_product_list: bool = False, # The list of products that give access to a member_type or member_group inc_product_list: bool = False, # The list of products that give access to a member_type or member_group
inc_user: bool = False, inc_user: bool = False,
) -> Membership_Person_Base: ) -> Membership_Person_Base:
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
if membership_person_id := redis_lookup_id_random(record_id_random=membership_person_id, table_name='membership_person'): pass if membership_person_id := redis_lookup_id_random(record_id_random=membership_person_id, table_name='membership_person'): pass

View File

@@ -52,8 +52,9 @@ def load_person_obj(
inc_post_comment_list: bool = False, inc_post_comment_list: bool = False,
inc_product: bool = False, inc_product: bool = False,
inc_user: bool = False, inc_user: bool = False,
inc_user_role_list: bool = False,
) -> Person_Base|dict|bool: ) -> Person_Base|dict|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass
@@ -62,15 +63,20 @@ def load_person_obj(
if person_rec := sql_select(table_name='v_person', record_id=person_id): pass if person_rec := sql_select(table_name='v_person', record_id=person_id): pass
else: return False else: return False
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(person_rec)
try: try:
person_obj = Person_Base(**person_rec) person_obj = Person_Base(**person_rec)
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(person_obj) log.debug(person_obj)
except ValidationError as e: except ValidationError as e:
log.error(e.json()) log.error(e.json())
return False
# Updated 2021-07-09 # Updated 2021-07-09
if inc_contact: if inc_contact:
log.info('Need to include contact data...')
contact_id = person_rec.get('contact_id', None) contact_id = person_rec.get('contact_id', None)
log.debug(contact_id) log.debug(contact_id)
if contact_result := load_contact_obj( if contact_result := load_contact_obj(
@@ -116,6 +122,7 @@ def load_person_obj(
# Updated 2021-07-09 # Updated 2021-07-09
if inc_membership_person: if inc_membership_person:
log.info('Need to include membership person data...')
from app.methods.membership_person_methods import load_membership_person_obj from app.methods.membership_person_methods import load_membership_person_obj
membership_person_id = person_rec.get('membership_person_id', None) membership_person_id = person_rec.get('membership_person_id', None)
log.debug(membership_person_id) log.debug(membership_person_id)
@@ -174,6 +181,7 @@ def load_person_obj(
# Updated 2021-06-18 # Updated 2021-06-18
if inc_order_list: if inc_order_list:
log.info('Need to include order list data...')
if order_rec_list_result := get_order_rec_list( if order_rec_list_result := get_order_rec_list(
for_obj_type = 'person', for_obj_type = 'person',
for_obj_id = person_id, for_obj_id = person_id,
@@ -199,6 +207,7 @@ def load_person_obj(
# Updated 2021-06-18 # Updated 2021-06-18
if inc_organization: if inc_organization:
log.info('Need to include organization data...')
organization_id = person_rec.get('organization_id', None) organization_id = person_rec.get('organization_id', None)
log.debug(organization_id) log.debug(organization_id)
if organization_dict := load_organization_obj( if organization_dict := load_organization_obj(
@@ -214,22 +223,25 @@ def load_person_obj(
person_obj.organization = organization_dict person_obj.organization = organization_dict
else: person_obj.organization = None else: person_obj.organization = None
# Updated 2021-06-18 # Updated 2021-08-19
if inc_user: if inc_user:
# user_id = person_rec.get('user_id', None) log.info('Need to include user data...')
# log.debug(user_id) user_id = person_rec.get('user_id', None)
# if user_result := load_user_obj( log.debug(user_id)
# user_id = user_id, from app.methods.user_methods import load_user_obj
# limit = limit, if user_result := load_user_obj(
# by_alias = by_alias, user_id = user_id,
# exclude_unset = exclude_unset, limit = limit,
# model_as_dict = model_as_dict, by_alias = by_alias,
# enabled = enabled, exclude_unset = exclude_unset,
# ): model_as_dict = model_as_dict,
# person_obj.user = user_result enabled = enabled,
# else: person_obj.user = None inc_user_role_list = inc_user_role_list,
pass ):
person_obj.user = user_result
else: person_obj.user = None
print(person_obj)
if model_as_dict: if model_as_dict:
return person_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) # pylint: disable=no-member return person_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) # pylint: disable=no-member
else: else:
@@ -292,6 +304,41 @@ def get_person_rec_list(
# ### END ### API Person Methods ### get_person_rec_list() ### # ### END ### API Person Methods ### get_person_rec_list() ###
# ### BEGIN ### API Person Methods ### get_person_rec_w_external_id() ###
def get_person_rec_w_external_id(
account_id: str,
external_id: str,
enabled: str = 'enabled', # enabled, disabled, all
) -> dict|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
# else: return False
account_id = 99
data = {}
data['account_id'] = account_id
data['external_id'] = external_id
sql = f"""
SELECT `person`.id AS 'person_id', `person`.id_random AS 'person_id_random'
FROM `person` AS `person`
WHERE person.account_id = :account_id
AND person.external_id = :external_id
LIMIT 1;
"""
if person_rec_result := sql_select(data=data, sql=sql):
person_rec = person_rec_result
else:
person_rec = None
log.debug(person_rec_result)
return person_rec
# ### END ### API Person Methods ### get_person_rec_w_external_id() ###
# ### BEGIN ### API Person Methods ### create_person_obj() ### # ### BEGIN ### API Person Methods ### create_person_obj() ###
# NOTE: This will create a person and then also create a linked contact if person_obj.contact data is passed. The create_contact_obj will create a contact and then also create a linked address if person_obj.contact.address data is passed. # NOTE: This will create a person and then also create a linked contact if person_obj.contact data is passed. The create_contact_obj will create a contact and then also create a linked address if person_obj.contact.address data is passed.
# Reviewed and updated 2021-08-10 # Reviewed and updated 2021-08-10

View File

@@ -61,13 +61,16 @@ class Person_Base(BaseModel):
tagline: Optional[str] tagline: Optional[str]
birth_date: Optional[datetime.datetime] birth_date: Optional[datetime.date]
lu_gender_id: Optional[int] lu_gender_id: Optional[int]
lu_gender_name: Optional[str] lu_gender_name: Optional[str]
email_allowed: Optional[bool] email_allowed: Optional[bool]
paper_mail_allowed: Optional[bool] paper_mail_allowed: Optional[bool]
external_id: Optional[str]
external_import_id: Optional[str]
notes: Optional[str] notes: Optional[str]
created_on: Optional[datetime.datetime] = None created_on: Optional[datetime.datetime] = None

View File

@@ -24,6 +24,84 @@ from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter() router = APIRouter()
# First round: Working importing the Cvent Address Book export data
@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base)
async def importing_person_data_v2_external_id(
response: Response = Response,
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
account_id = 99
full_file_path = 'admin/temp/import_person_external_id.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str})
log.debug(df)
df_dict = df.to_dict(orient='records')
person_data_li = []
for record in df_dict:
person_id = None
if external_id := record.get('external_id', None): pass
elif external_id := record.get('External ID', None): pass
else:
log.info('No external ID was found.')
continue
if email := record.get('email', None): pass
elif email := record.get('Email Address', None): pass
else:
log.info('No email address was found.')
continue
data = {}
data['account_id'] = account_id
data['email'] = email
sql = f"""
SELECT *
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.email = :email
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1;
"""
if person_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing person record
log.debug('Found one record')
person_rec = person_rec_result
person_id = person_rec.get('person_id', None)
# contact_id = person_rec.get('contact_id', None)
# address_id = person_rec.get('address_id', None)
# user_id = person_rec.get('user_id', None)
# external_id = person_rec.get('external_id', None)
# external_import_id = person_rec.get('external_import_id', None)
person_data = {}
person_data['id'] = person_id
person_data['external_id'] = external_id
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
log.debug(person_obj_up_result)
else:
log.warning(person_obj_up_result)
continue # Something unexpected may have happened
person_rec = person_rec_result
else:
continue # Something unexpected may have happened
person_data_min = {}
person_data_min['person_id'] = person_id
person_data_min['full_name'] = person_rec.get('full_name', None)
person_data_min['email'] = email
person_data_li.append(person_data_min)
log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}")
return mk_resp(data=person_data_li)
# First round: Working importing the Cvent Address Book export data # First round: Working importing the Cvent Address Book export data
@router.post('/person_data_v2', response_model=Resp_Body_Base) @router.post('/person_data_v2', response_model=Resp_Body_Base)
async def importing_person_data_v2( async def importing_person_data_v2(
@@ -48,7 +126,7 @@ async def importing_person_data_v2(
account_id = 99 account_id = 99
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
#df = df.fillna('') # replace NaN with '' #df = df.fillna('') # replace NaN with ''
# df = df.fillna(None) # df = df.fillna(None)
# df = df.fillna('', inplace=True) # df = df.fillna('', inplace=True)
@@ -72,11 +150,11 @@ async def importing_person_data_v2(
# membership_person_id = None # membership_person_id = None
person_data = {} person_data = {}
if record['external_id']: if record['external_import_id']:
person_data['external_id'] = record['external_id'] person_data['external_import_id'] = record['external_import_id']
else: else:
person_data['external_id'] = record['email'] person_data['external_import_id'] = record['email']
external_id = person_data['external_id'] external_import_id = person_data['external_import_id']
if nickname := record.get('nickname', None): person_data['informal_name'] = nickname if nickname := record.get('nickname', None): person_data['informal_name'] = nickname
if given_name := record.get('given_name', None): person_data['given_name'] = given_name if given_name := record.get('given_name', None): person_data['given_name'] = given_name
@@ -160,12 +238,12 @@ async def importing_person_data_v2(
# continue # continue
data = {} data = {}
data['account_id'] = account_id data['account_id'] = account_id
data['external_id'] = external_id data['external_import_id'] = external_import_id
sql = f""" sql = f"""
SELECT * SELECT *
FROM `v_person` AS `person` FROM `v_person` AS `person`
WHERE person.account_id = :account_id WHERE person.account_id = :account_id
AND person.external_id = :external_id AND person.external_import_id = :external_import_id
ORDER BY `person`.created_on DESC, `person`.updated_on DESC ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1; LIMIT 1;
""" """
@@ -447,7 +525,7 @@ async def importing_person_data_v2_membership(
account_id = 99 account_id = 99
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df) log.debug(df)
df_dict = df.to_dict(orient='records') df_dict = df.to_dict(orient='records')
@@ -466,23 +544,23 @@ async def importing_person_data_v2_membership(
person_data = {} person_data = {}
person_data['account_id'] = account_id person_data['account_id'] = account_id
if record['external_id']: if record['external_import_id']:
person_data['external_id'] = record['external_id'] person_data['external_import_id'] = record['external_import_id']
else: else:
person_data['external_id'] = record['email'] person_data['external_import_id'] = record['email']
external_id = person_data['external_id'] external_import_id = person_data['external_import_id']
if membership_type_name := record.get('membership_type', None): pass if membership_type_name := record.get('membership_type', None): pass
else: continue else: continue
data = {} data = {}
data['account_id'] = account_id data['account_id'] = account_id
data['external_id'] = external_id data['external_import_id'] = external_import_id
sql = f""" sql = f"""
SELECT * SELECT *
FROM `v_person` AS `person` FROM `v_person` AS `person`
WHERE person.account_id = :account_id WHERE person.account_id = :account_id
AND person.external_id = :external_id AND person.external_import_id = :external_import_id
ORDER BY `person`.created_on DESC, `person`.updated_on DESC ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1; LIMIT 1;
""" """
@@ -661,7 +739,7 @@ async def importing_person_data(
account_id = 99 account_id = 99
full_file_path = 'admin/temp/import_person_data.xlsx' full_file_path = 'admin/temp/import_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
#df = df.fillna('') # replace NaN with '' #df = df.fillna('') # replace NaN with ''
# df = df.fillna(None) # df = df.fillna(None)
# df = df.fillna('', inplace=True) # df = df.fillna('', inplace=True)
@@ -683,10 +761,10 @@ async def importing_person_data(
user_id = None user_id = None
person_data = {} person_data = {}
if record['external_id']: if record['external_import_id']:
person_data['external_id'] = record['external_id'] person_data['external_import_id'] = record['external_import_id']
else: else:
person_data['external_id'] = record['email'] person_data['external_import_id'] = record['email']
# person_data['informal_name'] = record['informal_name'] # person_data['informal_name'] = record['informal_name']
person_data['given_name'] = record['given_name'] person_data['given_name'] = record['given_name']
if record['middle_name']: if record['middle_name']:
@@ -732,14 +810,14 @@ async def importing_person_data(
person_data['meta_json'] = json.dumps(meta_data, indent=4) person_data['meta_json'] = json.dumps(meta_data, indent=4)
# Look up by email address or external ID and INSERT or UPDATE new person record # Look up by email address or external import ID and INSERT or UPDATE new person record
# INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a contact record and address record if needed
# INSERT or UPDATE a user record if needed # INSERT or UPDATE a user record if needed
# Process the person data # Process the person data
log.debug(person_data) log.debug(person_data)
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
# continue # continue
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']): if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']):
if not isinstance(person_rec_li_result, list): if not isinstance(person_rec_li_result, list):
# Pull out IDs and UPDATE existing person record # Pull out IDs and UPDATE existing person record
log.debug('Found one record') log.debug('Found one record')
@@ -982,7 +1060,7 @@ async def importing_cont_edu_cert_person_data(
cont_edu_cert_id = 3 cont_edu_cert_id = 3
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df) log.debug(df)
df_dict = df.to_dict(orient='records') df_dict = df.to_dict(orient='records')
@@ -1114,7 +1192,7 @@ async def importing_cont_edu_cert_person_data_touch(
cont_edu_cert_id = 3 cont_edu_cert_id = 3
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df) log.debug(df)
df_dict = df.to_dict(orient='records') df_dict = df.to_dict(orient='records')

View File

@@ -214,7 +214,7 @@ async def get_person_obj_membership_person(
else: return mk_resp(data=None, status_code=404) else: return mk_resp(data=None, status_code=404)
if membership_person_rec := sql_select(table_name='v_membership_person', field_name='person_id', field_value=person_id): pass if membership_person_rec := sql_select(table_name='v_membership_person', field_name='person_id', field_value=person_id): pass
else: return False else: return mk_resp(data=None, status_code=404)
membership_person_id = membership_person_rec.get('membership_person_id', None) membership_person_id = membership_person_rec.get('membership_person_id', None)
log.debug(membership_person_id) log.debug(membership_person_id)
@@ -243,7 +243,7 @@ async def get_person_obj_membership_person(
# inc_user = inc_user, # inc_user = inc_user,
): ):
response_data = membership_person_result response_data = membership_person_result
else: return mk_resp(data=False, status_code=400, response=response) # Bad Request else: return mk_resp(data=False, status_code=404, response=response) # Bad Request
return mk_resp(data=response_data) return mk_resp(data=response_data)
# ### END ### API Membership Person ### get_person_obj_membership_person() ### # ### END ### API Membership Person ### get_person_obj_membership_person() ###

View File

@@ -11,7 +11,7 @@ from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_
# from app.methods.membership_person_methods import load_membership_person_obj # from app.methods.membership_person_methods import load_membership_person_obj
from app.methods.order_methods import get_order_rec_list, load_order_obj from app.methods.order_methods import get_order_rec_list, load_order_obj
from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj from app.methods.person_methods import create_update_person_obj, get_person_rec_list, get_person_rec_w_external_id, load_person_obj, update_person_obj
from app.models.person_models import Person_Base from app.models.person_models import Person_Base
from app.models.response_models import Resp_Body_Base, mk_resp from app.models.response_models import Resp_Body_Base, mk_resp
@@ -179,6 +179,57 @@ async def get_person_obj_li(
return result return result
# ### BEGIN ### API Person ### person_obj_external_id() ###
# NOTE: This endpoint is a little different. It will likely be used to help log in the person/user coming from an external service (Cvent).
# Updated 2021-08-19
@router.get('/person/external_id', response_model=Resp_Body_Base)
async def person_obj_external_id(
account_id: str = Query(..., min_length=1, max_length=22),
external_id: str = Query(..., min_length=1, max_length=75),
limit: int = 500,
enabled: str = 'enabled',
inc_address: bool = False,
inc_contact: bool = False,
# inc_person: bool = False,
inc_user: bool = False,
inc_user_role_list: bool = False,
x_account_id: str = Header(...),
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
response: Response = Response,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
# else: return mk_resp(data=None, status_code=404, response=response)
account_id = 99 # WARNING!!!! Get rid of 99!
if person_data := get_person_rec_w_external_id(account_id=account_id, external_id=external_id): pass
else: return mk_resp(data=None, status_code=404, response=response)
person_id = person_data.get('person_id')
if person_dict := load_person_obj(
person_id = person_id,
limit = limit,
model_as_dict = True, # NOTE: returning model as a dict
enabled = enabled,
inc_address = inc_address,
inc_contact = inc_contact,
inc_user = inc_user,
inc_user_role_list = inc_user_role_list,
):
if isinstance(person_dict, dict):
response_data = person_dict
else:
response_data = person_dict
else:
return mk_resp(data=False, status_code=400, response=response) # Bad Request
return mk_resp(data=response_data)
# ### END ### API Person ### person_obj_external_id() ###
# ### BEGIN ### API Person ### get_person_obj() ### # ### BEGIN ### API Person ### get_person_obj() ###
# Working well as of 2021-06-25. Using as a template for other routes. # Working well as of 2021-06-25. Using as a template for other routes.
@router.get('/person/{person_id}', response_model=Resp_Body_Base) @router.get('/person/{person_id}', response_model=Resp_Body_Base)
@@ -216,7 +267,7 @@ async def get_person_obj(
exclude_unset: Optional[bool] = True, exclude_unset: Optional[bool] = True,
response: Response = Response, response: Response = Response,
): ):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass