From 4eaeb152fe51fe331862eb1af0ffb7ea167b245d Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 19 Aug 2021 15:10:59 -0400 Subject: [PATCH] Working on data importing. Mainly for IDAA membership. --- app/methods/membership_person_methods.py | 2 +- app/methods/person_methods.py | 77 +++++++++++--- app/models/person_models.py | 5 +- app/routers/importing.py | 122 +++++++++++++++++++---- app/routers/membership_person.py | 4 +- app/routers/person.py | 55 +++++++++- 6 files changed, 222 insertions(+), 43 deletions(-) diff --git a/app/methods/membership_person_methods.py b/app/methods/membership_person_methods.py index f307d2f..ce4b1c8 100644 --- a/app/methods/membership_person_methods.py +++ b/app/methods/membership_person_methods.py @@ -46,7 +46,7 @@ def load_membership_person_obj( inc_product_list: bool = False, # The list of products that give access to a member_type or member_group inc_user: bool = False, ) -> Membership_Person_Base: - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if membership_person_id := redis_lookup_id_random(record_id_random=membership_person_id, table_name='membership_person'): pass diff --git a/app/methods/person_methods.py b/app/methods/person_methods.py index 8ced9e6..4ea2505 100644 --- a/app/methods/person_methods.py +++ b/app/methods/person_methods.py @@ -52,8 +52,9 @@ def load_person_obj( inc_post_comment_list: bool = False, inc_product: bool = False, inc_user: bool = False, + inc_user_role_list: bool = False, ) -> Person_Base|dict|bool: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass @@ -62,15 +63,20 @@ def load_person_obj( if person_rec := sql_select(table_name='v_person', record_id=person_id): pass else: return False + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(person_rec) + try: person_obj = Person_Base(**person_rec) log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(person_obj) except ValidationError as e: log.error(e.json()) + return False # Updated 2021-07-09 if inc_contact: + log.info('Need to include contact data...') contact_id = person_rec.get('contact_id', None) log.debug(contact_id) if contact_result := load_contact_obj( @@ -116,6 +122,7 @@ def load_person_obj( # Updated 2021-07-09 if inc_membership_person: + log.info('Need to include membership person data...') from app.methods.membership_person_methods import load_membership_person_obj membership_person_id = person_rec.get('membership_person_id', None) log.debug(membership_person_id) @@ -174,6 +181,7 @@ def load_person_obj( # Updated 2021-06-18 if inc_order_list: + log.info('Need to include order list data...') if order_rec_list_result := get_order_rec_list( for_obj_type = 'person', for_obj_id = person_id, @@ -199,6 +207,7 @@ def load_person_obj( # Updated 2021-06-18 if inc_organization: + log.info('Need to include organization data...') organization_id = person_rec.get('organization_id', None) log.debug(organization_id) if organization_dict := load_organization_obj( @@ -214,22 +223,25 @@ def load_person_obj( person_obj.organization = organization_dict else: person_obj.organization = None - # Updated 2021-06-18 + # Updated 2021-08-19 if inc_user: - # user_id = person_rec.get('user_id', None) - # log.debug(user_id) - # if user_result := load_user_obj( - # user_id = user_id, - # limit = limit, - # by_alias = by_alias, - # exclude_unset = exclude_unset, - # model_as_dict = model_as_dict, - # enabled = enabled, - # ): - # person_obj.user = user_result - # else: person_obj.user = None - pass + log.info('Need to include user data...') + user_id = person_rec.get('user_id', None) + log.debug(user_id) + from app.methods.user_methods import load_user_obj + if user_result := load_user_obj( + user_id = user_id, + limit = limit, + by_alias = by_alias, + exclude_unset = exclude_unset, + model_as_dict = model_as_dict, + enabled = enabled, + inc_user_role_list = inc_user_role_list, + ): + person_obj.user = user_result + else: person_obj.user = None + print(person_obj) if model_as_dict: return person_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) # pylint: disable=no-member else: @@ -292,6 +304,41 @@ def get_person_rec_list( # ### END ### API Person Methods ### get_person_rec_list() ### +# ### BEGIN ### API Person Methods ### get_person_rec_w_external_id() ### +def get_person_rec_w_external_id( + account_id: str, + external_id: str, + enabled: str = 'enabled', # enabled, disabled, all + ) -> dict|bool: + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + # if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass + # else: return False + account_id = 99 + + data = {} + data['account_id'] = account_id + data['external_id'] = external_id + + sql = f""" + SELECT `person`.id AS 'person_id', `person`.id_random AS 'person_id_random' + FROM `person` AS `person` + WHERE person.account_id = :account_id + AND person.external_id = :external_id + LIMIT 1; + """ + + if person_rec_result := sql_select(data=data, sql=sql): + person_rec = person_rec_result + else: + person_rec = None + log.debug(person_rec_result) + + return person_rec +# ### END ### API Person Methods ### get_person_rec_w_external_id() ### + + # ### BEGIN ### API Person Methods ### create_person_obj() ### # NOTE: This will create a person and then also create a linked contact if person_obj.contact data is passed. The create_contact_obj will create a contact and then also create a linked address if person_obj.contact.address data is passed. # Reviewed and updated 2021-08-10 diff --git a/app/models/person_models.py b/app/models/person_models.py index 08c1cd5..753ffd4 100644 --- a/app/models/person_models.py +++ b/app/models/person_models.py @@ -61,13 +61,16 @@ class Person_Base(BaseModel): tagline: Optional[str] - birth_date: Optional[datetime.datetime] + birth_date: Optional[datetime.date] lu_gender_id: Optional[int] lu_gender_name: Optional[str] email_allowed: Optional[bool] paper_mail_allowed: Optional[bool] + external_id: Optional[str] + external_import_id: Optional[str] + notes: Optional[str] created_on: Optional[datetime.datetime] = None diff --git a/app/routers/importing.py b/app/routers/importing.py index 57395fb..f74e18b 100644 --- a/app/routers/importing.py +++ b/app/routers/importing.py @@ -24,6 +24,84 @@ from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() +# First round: Working importing the Cvent Address Book export data +@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base) +async def importing_person_data_v2_external_id( + response: Response = Response, + ): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + account_id = 99 + full_file_path = 'admin/temp/import_person_external_id.xlsx' + + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str}) + log.debug(df) + + df_dict = df.to_dict(orient='records') + + person_data_li = [] + + for record in df_dict: + person_id = None + if external_id := record.get('external_id', None): pass + elif external_id := record.get('External ID', None): pass + else: + log.info('No external ID was found.') + continue + + if email := record.get('email', None): pass + elif email := record.get('Email Address', None): pass + else: + log.info('No email address was found.') + continue + + data = {} + data['account_id'] = account_id + data['email'] = email + sql = f""" + SELECT * + FROM `v_person` AS `person` + WHERE person.account_id = :account_id + AND person.email = :email + ORDER BY `person`.created_on DESC, `person`.updated_on DESC + LIMIT 1; + """ + + if person_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing person record + log.debug('Found one record') + person_rec = person_rec_result + person_id = person_rec.get('person_id', None) + # contact_id = person_rec.get('contact_id', None) + # address_id = person_rec.get('address_id', None) + # user_id = person_rec.get('user_id', None) + # external_id = person_rec.get('external_id', None) + # external_import_id = person_rec.get('external_import_id', None) + + person_data = {} + person_data['id'] = person_id + person_data['external_id'] = external_id + + if person_obj_up_result := sql_update(data=person_data, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + continue # Something unexpected may have happened + person_rec = person_rec_result + else: + continue # Something unexpected may have happened + + person_data_min = {} + person_data_min['person_id'] = person_id + person_data_min['full_name'] = person_rec.get('full_name', None) + person_data_min['email'] = email + person_data_li.append(person_data_min) + log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}") + + return mk_resp(data=person_data_li) + + # First round: Working importing the Cvent Address Book export data @router.post('/person_data_v2', response_model=Resp_Body_Base) async def importing_person_data_v2( @@ -48,7 +126,7 @@ async def importing_person_data_v2( account_id = 99 full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) @@ -72,11 +150,11 @@ async def importing_person_data_v2( # membership_person_id = None person_data = {} - if record['external_id']: - person_data['external_id'] = record['external_id'] + if record['external_import_id']: + person_data['external_import_id'] = record['external_import_id'] else: - person_data['external_id'] = record['email'] - external_id = person_data['external_id'] + person_data['external_import_id'] = record['email'] + external_import_id = person_data['external_import_id'] if nickname := record.get('nickname', None): person_data['informal_name'] = nickname if given_name := record.get('given_name', None): person_data['given_name'] = given_name @@ -160,12 +238,12 @@ async def importing_person_data_v2( # continue data = {} data['account_id'] = account_id - data['external_id'] = external_id + data['external_import_id'] = external_import_id sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id - AND person.external_id = :external_id + AND person.external_import_id = :external_import_id ORDER BY `person`.created_on DESC, `person`.updated_on DESC LIMIT 1; """ @@ -447,7 +525,7 @@ async def importing_person_data_v2_membership( account_id = 99 full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) log.debug(df) df_dict = df.to_dict(orient='records') @@ -466,23 +544,23 @@ async def importing_person_data_v2_membership( person_data = {} person_data['account_id'] = account_id - if record['external_id']: - person_data['external_id'] = record['external_id'] + if record['external_import_id']: + person_data['external_import_id'] = record['external_import_id'] else: - person_data['external_id'] = record['email'] - external_id = person_data['external_id'] + person_data['external_import_id'] = record['email'] + external_import_id = person_data['external_import_id'] if membership_type_name := record.get('membership_type', None): pass else: continue data = {} data['account_id'] = account_id - data['external_id'] = external_id + data['external_import_id'] = external_import_id sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id - AND person.external_id = :external_id + AND person.external_import_id = :external_import_id ORDER BY `person`.created_on DESC, `person`.updated_on DESC LIMIT 1; """ @@ -661,7 +739,7 @@ async def importing_person_data( account_id = 99 full_file_path = 'admin/temp/import_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) @@ -683,10 +761,10 @@ async def importing_person_data( user_id = None person_data = {} - if record['external_id']: - person_data['external_id'] = record['external_id'] + if record['external_import_id']: + person_data['external_import_id'] = record['external_import_id'] else: - person_data['external_id'] = record['email'] + person_data['external_import_id'] = record['email'] # person_data['informal_name'] = record['informal_name'] person_data['given_name'] = record['given_name'] if record['middle_name']: @@ -732,14 +810,14 @@ async def importing_person_data( person_data['meta_json'] = json.dumps(meta_data, indent=4) - # Look up by email address or external ID and INSERT or UPDATE new person record + # Look up by email address or external import ID and INSERT or UPDATE new person record # INSERT or UPDATE a contact record and address record if needed # INSERT or UPDATE a user record if needed # Process the person data log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue - if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']): + if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']): if not isinstance(person_rec_li_result, list): # Pull out IDs and UPDATE existing person record log.debug('Found one record') @@ -982,7 +1060,7 @@ async def importing_cont_edu_cert_person_data( cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) log.debug(df) df_dict = df.to_dict(orient='records') @@ -1114,7 +1192,7 @@ async def importing_cont_edu_cert_person_data_touch( cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) log.debug(df) df_dict = df.to_dict(orient='records') diff --git a/app/routers/membership_person.py b/app/routers/membership_person.py index 2f5ded6..5a71288 100644 --- a/app/routers/membership_person.py +++ b/app/routers/membership_person.py @@ -214,7 +214,7 @@ async def get_person_obj_membership_person( else: return mk_resp(data=None, status_code=404) if membership_person_rec := sql_select(table_name='v_membership_person', field_name='person_id', field_value=person_id): pass - else: return False + else: return mk_resp(data=None, status_code=404) membership_person_id = membership_person_rec.get('membership_person_id', None) log.debug(membership_person_id) @@ -243,7 +243,7 @@ async def get_person_obj_membership_person( # inc_user = inc_user, ): response_data = membership_person_result - else: return mk_resp(data=False, status_code=400, response=response) # Bad Request + else: return mk_resp(data=False, status_code=404, response=response) # Bad Request return mk_resp(data=response_data) # ### END ### API Membership Person ### get_person_obj_membership_person() ### diff --git a/app/routers/person.py b/app/routers/person.py index cb0826b..b9469ce 100644 --- a/app/routers/person.py +++ b/app/routers/person.py @@ -11,7 +11,7 @@ from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_ # from app.methods.membership_person_methods import load_membership_person_obj from app.methods.order_methods import get_order_rec_list, load_order_obj -from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj +from app.methods.person_methods import create_update_person_obj, get_person_rec_list, get_person_rec_w_external_id, load_person_obj, update_person_obj from app.models.person_models import Person_Base from app.models.response_models import Resp_Body_Base, mk_resp @@ -179,6 +179,57 @@ async def get_person_obj_li( return result +# ### BEGIN ### API Person ### person_obj_external_id() ### +# NOTE: This endpoint is a little different. It will likely be used to help log in the person/user coming from an external service (Cvent). +# Updated 2021-08-19 +@router.get('/person/external_id', response_model=Resp_Body_Base) +async def person_obj_external_id( + account_id: str = Query(..., min_length=1, max_length=22), + external_id: str = Query(..., min_length=1, max_length=75), + limit: int = 500, + enabled: str = 'enabled', + inc_address: bool = False, + inc_contact: bool = False, + # inc_person: bool = False, + inc_user: bool = False, + inc_user_role_list: bool = False, + x_account_id: str = Header(...), + by_alias: Optional[bool] = True, + exclude_unset: Optional[bool] = True, + response: Response = Response, + ): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + # if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass + # else: return mk_resp(data=None, status_code=404, response=response) + account_id = 99 # WARNING!!!! Get rid of 99! + + if person_data := get_person_rec_w_external_id(account_id=account_id, external_id=external_id): pass + else: return mk_resp(data=None, status_code=404, response=response) + person_id = person_data.get('person_id') + + if person_dict := load_person_obj( + person_id = person_id, + limit = limit, + model_as_dict = True, # NOTE: returning model as a dict + enabled = enabled, + inc_address = inc_address, + inc_contact = inc_contact, + inc_user = inc_user, + inc_user_role_list = inc_user_role_list, + ): + if isinstance(person_dict, dict): + response_data = person_dict + else: + response_data = person_dict + else: + return mk_resp(data=False, status_code=400, response=response) # Bad Request + + return mk_resp(data=response_data) +# ### END ### API Person ### person_obj_external_id() ### + + # ### BEGIN ### API Person ### get_person_obj() ### # Working well as of 2021-06-25. Using as a template for other routes. @router.get('/person/{person_id}', response_model=Resp_Body_Base) @@ -216,7 +267,7 @@ async def get_person_obj( exclude_unset: Optional[bool] = True, response: Response = Response, ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass