From 4752c81e7f492f8863396b0457b5db75788d58ac Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 15 Mar 2022 11:59:40 -0400 Subject: [PATCH] Updates for IDAA Cvent API sync. Less debugging turned on. --- app/methods/e_cvent_methods.py | 1 + app/methods/membership_person_methods.py | 18 +++++----- app/methods/person_methods.py | 7 ++-- app/models/person_models.py | 4 +-- app/routers/c_idaa.py | 33 +++++++++++++++-- app/routers/importing.py | 46 ++++++++++++------------ 6 files changed, 70 insertions(+), 39 deletions(-) diff --git a/app/methods/e_cvent_methods.py b/app/methods/e_cvent_methods.py index 7383097..bfbed4a 100644 --- a/app/methods/e_cvent_methods.py +++ b/app/methods/e_cvent_methods.py @@ -485,6 +485,7 @@ def create_update_aether_person( person_data = {} person_data['external_id'] = person_external_id + person_data['external_sys_id'] = cvent_contact_obj.get('sourceId') # person_data['pronouns'] = ??? person_data['informal_name'] = cvent_contact_obj.get('nickname') person_data['title_names'] = cvent_contact_obj.get('prefix') diff --git a/app/methods/membership_person_methods.py b/app/methods/membership_person_methods.py index ec16139..1c05d5d 100644 --- a/app/methods/membership_person_methods.py +++ b/app/methods/membership_person_methods.py @@ -69,7 +69,6 @@ def load_membership_person_obj( log.error(e.json()) return False - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(membership_person_obj) # Updated 2021-06-24 @@ -269,13 +268,14 @@ def load_membership_person_obj( # ### BEGIN ### API Membership Person Methods ### get_membership_person_rec_list() ### +@logger_reset def get_membership_person_rec_list( for_obj_type: str, for_obj_id: str, limit: int = 1000, enabled: str = 'enabled', # enabled, disabled, all ) -> list|bool: - # log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if for_obj_id := redis_lookup_id_random(record_id_random=for_obj_id, table_name=for_obj_type): pass @@ -379,7 +379,7 @@ def get_membership_person_rec_list( membership_person_rec_li = membership_person_rec_li_result else: membership_person_rec_li = [] - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(membership_person_rec_li_result) return membership_person_rec_li @@ -388,6 +388,7 @@ def get_membership_person_rec_list( # ### BEGIN ### API Membership Person Methods ### create_membership_person_obj() ### # Updated 2022-01-04 +@logger_reset def create_membership_person_obj( account_id: int, person_id: int, @@ -396,7 +397,7 @@ def create_membership_person_obj( fail_any: bool = False, # Fail if any thing goes wrong for sub objects return_outline: bool = False, ) -> bool|dict|int: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # ### SECTION ### Secondary data validation @@ -425,7 +426,6 @@ def create_membership_person_obj( membership_person_obj = membership_person_dict_obj membership_person_obj.account_id = account_id membership_person_obj.person_id = person_id - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(membership_person_obj) membership_person_dict = membership_person_obj.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'membership_cfg', 'membership_person_group_list', 'membership_person_profile', 'membership_person_type', 'membership_group', 'membership_group_list', 'membership_type', 'membership_type_list', 'person', 'product', 'created_on', 'updated_on'}) @@ -505,6 +505,7 @@ def create_membership_person_obj( # ### BEGIN ### API Membership Person Methods ### update_membership_person_obj() ### # Updated 2022-01-04 +@logger_reset def update_membership_person_obj( membership_person_id: int, membership_person_dict_obj: Membership_Person_Base, @@ -512,7 +513,7 @@ def update_membership_person_obj( fail_any: bool = False, # Fail if any thing goes wrong for sub objects return_outline: bool = False, ) -> bool|dict: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # ### SECTION ### Secondary data validation @@ -536,11 +537,9 @@ def update_membership_person_obj( else: membership_person_obj = membership_person_dict_obj membership_person_obj.id = membership_person_id - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(json.dumps(membership_person_obj, indent=2, default=str)) membership_person_dict = membership_person_obj.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'membership_cfg', 'membership_person_group_list', 'membership_person_profile', 'membership_person_type', 'membership_group', 'membership_group_list', 'membership_type', 'membership_type_list', 'person', 'product', 'created_on', 'updated_on'}) - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(membership_person_dict) # ### SECTION ### Process data @@ -642,6 +641,7 @@ def update_membership_person_obj( # ### BEGIN ### API Membership Person Methods ### create_update_membership_person_obj() ### # Updated 2021-08-25 +@logger_reset def create_update_membership_person_obj( membership_person_dict_obj: Membership_Person_Base|dict, person_id: int|str, @@ -651,7 +651,7 @@ def create_update_membership_person_obj( fail_any: bool = False, # Fail if any thing goes wrong for sub objects return_outline: bool = False, ) -> int|bool: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) log.info('Checking requirements...') diff --git a/app/methods/person_methods.py b/app/methods/person_methods.py index 3c2fa1c..92b09bd 100644 --- a/app/methods/person_methods.py +++ b/app/methods/person_methods.py @@ -965,12 +965,13 @@ def create_update_person_obj_v4b( # ### BEGIN ### API Person Methods ### get_person_rec_w_external_id() ### # Updated 2021-08-19 +@logger_reset def get_person_rec_w_external_id( account_id: str, external_id: str, enabled: str = 'enabled', # enabled, disabled, all ) -> dict|bool: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass @@ -1001,10 +1002,11 @@ def get_person_rec_w_external_id( # ### BEGIN ### API Person Methods ### get_account_id_w_person_id() ### # Updated 2021-08-25 +@logger_reset def get_account_id_w_person_id( person_id: int|str, ) -> bool|int|None: - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass @@ -1020,7 +1022,6 @@ def get_account_id_w_person_id( LIMIT 1; """ - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL if person_data_result := sql_select(data=data, sql=sql): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(person_data_result) diff --git a/app/models/person_models.py b/app/models/person_models.py index aba7844..67a6802 100644 --- a/app/models/person_models.py +++ b/app/models/person_models.py @@ -91,8 +91,8 @@ class Person_Base(BaseModel): email_allowed: Optional[bool] paper_mail_allowed: Optional[bool] - external_id: Optional[str] - external_import_id: Optional[str] + external_id: Optional[str] # Generated internally or externally. Needs to be stable. It should not change. + external_sys_id: Optional[str] # Generated by external system (should be stable and not change) stripe_customer_id: Optional[str] diff --git a/app/routers/c_idaa.py b/app/routers/c_idaa.py index bcd5915..1267395 100644 --- a/app/routers/c_idaa.py +++ b/app/routers/c_idaa.py @@ -24,12 +24,27 @@ async def refresh_membership_status( log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) + sql = f""" + UPDATE user + INNER JOIN person ON user.id = person.user_id + INNER JOIN membership_person ON person.id = membership_person.person_id + SET user.email_verified = 1, user.enable = 1, user.enable_from = membership_person.start_on, user.enable_to = membership_person.end_on, user.verified = 1, user.group = 'pending_member' + WHERE user.account_id = 13 AND membership_person.end_on >= NOW() + AND (person.external_sys_id IS NULL OR person.external_sys_id = ''); + """ + if user_update_result := sql_update(sql=sql): + # Pull out IDs of existing person record + log.debug(user_update_result) + else: + log.warning(f'The SQL UPDATE of user table records failed for pending members') + sql = f""" UPDATE user INNER JOIN person ON user.id = person.user_id INNER JOIN membership_person ON person.id = membership_person.person_id SET user.email_verified = 1, user.enable = 1, user.enable_from = membership_person.start_on, user.enable_to = membership_person.end_on, user.verified = 1, user.group = 'current_member' - WHERE user.account_id = 13 AND membership_person.end_on >= NOW(); + WHERE user.account_id = 13 AND membership_person.end_on >= NOW() + AND (person.external_sys_id IS NOT NULL AND person.external_sys_id != ''); """ if user_update_result := sql_update(sql=sql): # Pull out IDs of existing person record @@ -77,11 +92,25 @@ async def refresh_membership_status( log.warning(f'The SQL UPDATE of user table records failed for not members') + sql = f""" + UPDATE person + INNER JOIN membership_person ON person.id = membership_person.person_id + SET person.enable = 1, person.group = 'pending_member' + WHERE person.account_id = 13 AND membership_person.end_on >= NOW() + AND (person.external_sys_id IS NULL OR person.external_sys_id = ''); + """ + if person_update_result := sql_update(sql=sql): + # Pull out IDs of existing person record + log.debug(person_update_result) + else: + log.warning(f'The SQL UPDATE of person table records failed for current members') + sql = f""" UPDATE person INNER JOIN membership_person ON person.id = membership_person.person_id SET person.enable = 1, person.group = 'current_member' - WHERE person.account_id = 13 AND membership_person.end_on >= NOW(); + WHERE person.account_id = 13 AND membership_person.end_on >= NOW() + AND (person.external_sys_id IS NOT NULL AND person.external_sys_id != ''); """ if person_update_result := sql_update(sql=sql): # Pull out IDs of existing person record diff --git a/app/routers/importing.py b/app/routers/importing.py index 415e942..b51652b 100644 --- a/app/routers/importing.py +++ b/app/routers/importing.py @@ -123,7 +123,7 @@ async def importing_update_w_external_id( FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.email = :email - AND (external_id IS NULL OR external_import_id IS NULL) + AND (external_id IS NULL OR external_sys_id IS NULL) LIMIT 1; """ @@ -138,15 +138,15 @@ async def importing_update_w_external_id( # address_id = person_rec.get('address_id', None) # user_id = person_rec.get('user_id', None) # external_id = person_rec.get('external_id', None) - # external_import_id = person_rec.get('external_import_id', None) + # external_sys_id = person_rec.get('external_sys_id', None) person_data = {} person_data['id'] = person_id person_data['external_id'] = external_id if source_id: - person_data['external_import_id'] = source_id + person_data['external_sys_id'] = source_id else: - person_data['external_import_id'] = email + person_data['external_sys_id'] = email log.debug(person_data) if person_obj_up_result := sql_update(data=person_data, table_name='person'): @@ -337,11 +337,11 @@ async def ins_up_person_contact_address_user_data( person_data = {} person_data['external_id'] = external_id if source_id: - person_data['external_import_id'] = source_id + person_data['external_sys_id'] = source_id else: - person_data['external_import_id'] = email - external_import_id = person_data['external_import_id'] - log.debug(external_import_id) + person_data['external_sys_id'] = email + external_sys_id = person_data['external_sys_id'] + log.debug(external_sys_id) if record.get('informal_name'): person_data['informal_name'] = record.get('informal_name') if record.get('given_name'): person_data['given_name'] = record.get('given_name') @@ -434,12 +434,12 @@ async def ins_up_person_contact_address_user_data( data = {} data['account_id'] = account_id data['external_id'] = external_id - data['external_import_id'] = external_import_id + data['external_sys_id'] = external_sys_id # sql = f""" # SELECT * # FROM `v_person` AS `person` # WHERE person.account_id = :account_id - # AND person.external_import_id = :external_import_id + # AND person.external_sys_id = :external_sys_id # LIMIT 1; # """ @@ -895,11 +895,11 @@ async def ins_up_membership_person_data( person_data = {} if source_id: - person_data['external_import_id'] = source_id + person_data['external_sys_id'] = source_id else: - person_data['external_import_id'] = email - external_import_id = person_data['external_import_id'] - log.debug(external_import_id) + person_data['external_sys_id'] = email + external_sys_id = person_data['external_sys_id'] + log.debug(external_sys_id) if membership_type_name := record.get('membership_type', None): pass else: @@ -909,13 +909,13 @@ async def ins_up_membership_person_data( data = {} data['account_id'] = account_id data['external_id'] = external_id - data['external_import_id'] = external_import_id + data['external_sys_id'] = external_sys_id sql = f""" SELECT * FROM `v_person` AS `person` WHERE person.account_id = :account_id AND person.external_id = :external_id - /*AND person.external_import_id = :external_import_id*/ + /*AND person.external_sys_id = :external_sys_id*/ LIMIT 1; """ @@ -1096,7 +1096,7 @@ async def importing_person_data( account_id = 99 full_file_path = 'admin/temp/import_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) #df = df.fillna('') # replace NaN with '' # df = df.fillna(None) # df = df.fillna('', inplace=True) @@ -1118,10 +1118,10 @@ async def importing_person_data( user_id = None person_data = {} - if record['external_import_id']: - person_data['external_import_id'] = record['external_import_id'] + if record['external_sys_id']: + person_data['external_sys_id'] = record['external_sys_id'] else: - person_data['external_import_id'] = record['email'] + person_data['external_sys_id'] = record['email'] # person_data['informal_name'] = record['informal_name'] person_data['given_name'] = record['given_name'] if record['middle_name']: @@ -1174,7 +1174,7 @@ async def importing_person_data( log.debug(person_data) # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') # continue - if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']): + if person_rec_li_result := sql_select(table_name='v_person', field_name='external_sys_id', field_value=person_data['external_sys_id']): if not isinstance(person_rec_li_result, list): # Pull out IDs and UPDATE existing person record log.debug('Found one record') @@ -1417,7 +1417,7 @@ async def importing_cont_edu_cert_person_data( cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) log.debug(df) df_dict = df.to_dict(orient='records') @@ -1549,7 +1549,7 @@ async def importing_cont_edu_cert_person_data_touch( cont_edu_cert_id = 3 full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx' - df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_sys_id': str, 'phone_home': str, 'phone_mobile': str, 'city': str, 'state_province': str, 'address_postal_code': str, 'country': str}) log.debug(df) df_dict = df.to_dict(orient='records')