From 6fc7905b04767f557b795c83d09736778804f652 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Wed, 9 Feb 2022 19:44:16 -0500 Subject: [PATCH] Work on Cvent API and IDAA --- app/methods/e_cvent_methods.py | 249 +++++++++++++++++++++++++-------- app/methods/user_methods.py | 28 ++-- app/routers/e_cvent.py | 24 +++- 3 files changed, 228 insertions(+), 73 deletions(-) diff --git a/app/methods/e_cvent_methods.py b/app/methods/e_cvent_methods.py index 62bc648..5578c1e 100644 --- a/app/methods/e_cvent_methods.py +++ b/app/methods/e_cvent_methods.py @@ -21,6 +21,7 @@ api['base_url'] = 'https://api-platform.cvent.com/ea' # Including /ea as the Cve api['headers'] = {} # { 'Content-Type': content_type, 'Authorization': 'Basic '+str(cvent_authorization_base64) } +# ### BEGIN ### API External Cvent Methods ### get_access_token() ### # Updated 2022-02-01 @logger_reset def get_access_token(): @@ -51,15 +52,29 @@ def get_access_token(): log.debug(f'Oauth Token Request Data:\n{api}') - resp = requests.post(url=uri, data=data, auth=HTTPBasicAuth(app['client_id'], app['secret'])) # Sending as HTML form data + try_request = True + limit = 0 + while try_request and limit < 3: + limit = limit + 1 - log.debug(f'Status Code: {resp.status_code}') - log.debug(f'Headers: {resp.headers}') - log.debug(f'Encoding: {resp.encoding}') - log.debug(f'JSON: {resp.json()}') - # log.debug('Text:', resp.text) + resp = requests.post(url=uri, data=data, auth=HTTPBasicAuth(app['client_id'], app['secret'])) # Sending as HTML form data - response_data = resp.json() + log.debug(f'Status Code: {resp.status_code}') + log.debug(f'Headers: {resp.headers}') + log.debug(f'Encoding: {resp.encoding}') + log.debug(f'JSON: {resp.json()}') + # log.debug('Text:', resp.text) + + response_data = resp.json() + log.debug(json.dumps(response_data, indent=2, default=str)) + + if 'message' in response_data and response_data['message'] == 'Too Many Requests': + log.warning('Hit Cvent rate limit. Sleeping for .5 seconds...') + time.sleep(.5) + elif 'access_token' in response_data: + access_token = response_data.get('access_token') + log.info(f'Got a new Access Token {access_token} from Cvent') + try_request = False api['access_token'] = response_data['access_token'] api['expires_in'] = response_data['expires_in'] @@ -73,16 +88,78 @@ def get_access_token(): log.debug(api) - log.warning('Sleeping for 1 second to avoid Cvent rate limit...') - time.sleep(1) + log.warning('Sleeping for .5 seconds to avoid Cvent rate limit...') + time.sleep(.5) return api +# ### END ### API External Cvent Methods ### get_access_token() ### +# ### BEGIN ### API External Cvent Methods ### get_group_contact_list() ### +# Updated 2022-02-09 +@logger_reset +def get_group_contact_list(contact_group_id: str=None): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + get_access_token() + + # Contact Group ID for IDAA Full Mailing List: UUID = '60ef4838-5340-4e2a-9d0a-b4ef310eaa69' + + endpoint = f'/contact-groups/{contact_group_id}/contacts' + uri = api['base_url']+endpoint + + params = {} + + try_request = True + limit = 0 + next_token = None + cvent_contact_list = [] + while try_request and limit < 100: + limit = limit + 1 + + if next_token: params['token'] = next_token + resp = requests.get(url=uri, headers=api['headers'], params=params) + + response_data = resp.json() + log.debug(json.dumps(response_data, indent=2, default=str)) + + if 'message' in response_data and response_data['message'] == 'Too Many Requests': + log.warning('Hit Cvent rate limit. Sleeping for .5 seconds...') + time.sleep(.5) + elif 'paging' in response_data: + log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) + + if total_count := response_data.get('paging').get('totalCount'): + log.info(f'Found {total_count} Cvent Contact results in the Cvent Group') + + cvent_contact_list = cvent_contact_list + response_data.get('data') + + if next_token := response_data.get('paging').get('nextToken'): + log.warning('Sleeping for .25 seconds to avoid Cvent rate limit...') + time.sleep(.25) + else: + try_request = False + else: + log.info('No Cvent Contact results for Cvent Group') + try_request = False + next_token = None + return None + else: + log.warning('Unexpected response from Cvent API') + return False + + if 'message' in response_data and response_data['message'] == 'Too Many Requests': return False + + return cvent_contact_list +# ### END ### API External Cvent Methods ### get_group_contact_list() ### + + +# ### BEGIN ### API External Cvent Methods ### get_contact_custom_field_list() ### # Updated 2022-01-31 @logger_reset def get_contact_custom_field_list(api): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) endpoint = '/custom-fields' @@ -104,8 +181,10 @@ def get_contact_custom_field_list(api): # f.write(pprint.pformat(response_data, indent=2, width=160)) return response_data +# ### END ### API External Cvent Methods ### get_contact_custom_field_list() ### +# ### BEGIN ### API External Cvent Methods ### get_contact_list() ### # Updated 2022-01-31 @logger_reset def get_contact_list(external_id: str=None, email: str=None): @@ -158,18 +237,20 @@ def get_contact_list(external_id: str=None, email: str=None): if 'message' in response_data and response_data['message'] == 'Too Many Requests': return False return response_data +# ### END ### API External Cvent Methods ### get_contact_list() ### -# ### BEGIN ### API Cvent Methods ### get_recent_contact_list() ### +# ### BEGIN ### API External Cvent Methods ### get_recent_contact_list() ### # Updated 2022-02-02 @logger_reset def get_recent_contact_list( from_created_on = datetime.datetime.utcnow() + datetime.timedelta(minutes=-60), - to_created_on = datetime.datetime.utcnow(), + to_created_on = None, # datetime.datetime.utcnow(), from_updated_on = datetime.datetime.utcnow() + datetime.timedelta(minutes=-60), to_updated_on = None, + get_only = 'created', # created, updated/modified ): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) get_access_token() @@ -191,54 +272,96 @@ def get_recent_contact_list( to_updated_on_str = to_updated_on.isoformat() log.info(f'To Updated On: {to_updated_on_str}') - if from_created_on and from_updated_on and to_created_on and to_updated_on: params['filter'] = f"deleted eq 'False' and (created gt '{from_created_on_str}Z' or lastModified gt '{from_updated_on_str}Z') and (created lt '{to_created_on_str}Z' or lastModified lt '{to_updated_on_str}Z')" elif from_created_on and to_created_on: params['filter'] = f"deleted eq 'False' and created gt '{from_created_on_str}Z' and created lt '{to_created_on_str}Z'" - elif updated_on: + elif from_updated_on and to_updated_on: + params['filter'] = f"deleted eq 'False' and lastModified gt '{from_updated_on_str}Z' and lastModified lt '{to_updated_on_str}Z'" + elif from_created_on: + params['filter'] = f"deleted eq 'False' and created gt '{from_created_on_str}Z'" + elif from_updated_on: params['filter'] = f"deleted eq 'False' and lastModified gt '{from_updated_on_str}Z'" else: - log.warning('Created On or Updated On is requried. Returing False') + log.warning('Created On or Updated On is required. Returning False') return False - resp = requests.get(url=uri, headers=api['headers'], params=params) - - response_data = resp.json() - # log.debug(json.dumps(response_data, indent=2, default=str)) - - cvent_contact_list = response_data.get('data') - - next_token = None - if 'paging' in response_data: - next_token = response_data.get('paging').get('nextToken') - log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) - + try_request = True limit = 0 - while next_token and limit < 100: + next_token = None + cvent_contact_list = [] + while try_request and limit < 100: limit = limit + 1 - params = {} - params['token'] = next_token + if next_token: params['token'] = next_token resp = requests.get(url=uri, headers=api['headers'], params=params) response_data = resp.json() - # log.debug(json.dumps(response_data, indent=2, default=str)) + log.debug(json.dumps(response_data, indent=2, default=str)) if 'message' in response_data and response_data['message'] == 'Too Many Requests': log.warning('Hit Cvent rate limit. Sleeping for .5 seconds...') time.sleep(.5) - else: - cvent_contact_list = cvent_contact_list + response_data.get('data') + elif 'paging' in response_data: + log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) - if 'paging' in response_data: - next_token = response_data.get('paging').get('nextToken') - log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) + if total_count := response_data.get('paging').get('totalCount'): + log.info(f'Found {total_count} Cvent Contact results') - log.warning('Sleeping for .25 seconds to avoid Cvent rate limit...') - time.sleep(.25) + cvent_contact_list = cvent_contact_list + response_data.get('data') + + if next_token := response_data.get('paging').get('nextToken'): + log.warning('Sleeping for .25 seconds to avoid Cvent rate limit...') + time.sleep(.25) + else: + try_request = False else: + log.info('No Cvent Contact results') + try_request = False next_token = None + return None + else: + log.warning('Unexpected response from Cvent API') + return False + + # cvent_contact_list = response_data.get('data') + + # next_token = None + # if 'paging' in response_data: + # if total_count := response_data.get('paging').get('totalCount'): + # log.info(f'Found {total_count} Cvent Contact results') + # next_token = response_data.get('paging').get('nextToken') + # log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) + # else: + # log.info('No Cvent Contact results') + # next_token = None + # return None + + # limit = 0 + # while next_token and limit < 100: + # limit = limit + 1 + + # params = {} + # params['token'] = next_token + # resp = requests.get(url=uri, headers=api['headers'], params=params) + + # response_data = resp.json() + # # log.debug(json.dumps(response_data, indent=2, default=str)) + + # if 'message' in response_data and response_data['message'] == 'Too Many Requests': + # log.warning('Hit Cvent rate limit. Sleeping for .5 seconds...') + # time.sleep(.5) + # else: + # cvent_contact_list = cvent_contact_list + response_data.get('data') + + # if 'paging' in response_data: + # next_token = response_data.get('paging').get('nextToken') + # log.debug(json.dumps(response_data.get('paging'), indent=2, default=str)) + + # log.warning('Sleeping for .25 seconds to avoid Cvent rate limit...') + # time.sleep(.25) + # else: + # next_token = None # log.debug(json.dumps(cvent_contact_list, indent=2, default=str)) # log.debug(json.dumps(response_data, indent=2, default=str)) @@ -246,10 +369,10 @@ def get_recent_contact_list( if 'message' in response_data and response_data['message'] == 'Too Many Requests': return False return cvent_contact_list -# ### END ### API Cvent Methods ### get_recent_contact_list() ### +# ### END ### API External Cvent Methods ### get_recent_contact_list() ### -# ### BEGIN ### API Cvent Methods ### get_contact_id() ### +# ### BEGIN ### API External Cvent Methods ### get_contact_id() ### # Updated 2022-02-01 @logger_reset def get_contact_id(contact_id: str): @@ -271,10 +394,10 @@ def get_contact_id(contact_id: str): if 'message' in response_data and response_data['message'] == 'Too Many Requests': return False return response_data -# ### END ### API Cvent Methods ### get_contact_id() ### +# ### END ### API External Cvent Methods ### get_contact_id() ### -# ### BEGIN ### API Cvent Methods ### modify_contact_id() ### +# ### BEGIN ### API External Cvent Methods ### modify_contact_id() ### # Updated 2022-02-01 @logger_reset def modify_contact_id(contact_id: str, field_list: list=[], custom_field_id: str=None, custom_field_value: str=None): @@ -304,22 +427,38 @@ def modify_contact_id(contact_id: str, field_list: list=[], custom_field_id: str data = field_list data['id'] = contact_id - if custom_field_id: - resp = requests.put(url=uri, headers=api['headers'], params=params, json=data) - else: - resp = requests.patch(url=uri, headers=api['headers'], params=params, json=data) + try_request = True + limit = 0 + while try_request and limit < 3: + limit = limit + 1 - log.info('Updated Cvent Contact ID: {contact_id}') - response_data = resp.json() - log.debug(json.dumps(response_data, indent=2, default=str)) + + if custom_field_id: + resp = requests.put(url=uri, headers=api['headers'], params=params, json=data) + else: + resp = requests.patch(url=uri, headers=api['headers'], params=params, json=data) + + response_data = resp.json() + log.debug(json.dumps(response_data, indent=2, default=str)) + + if 'message' in response_data and response_data['message'] == 'Too Many Requests': + log.warning('Hit Cvent rate limit. Sleeping for .5 seconds...') + time.sleep(.5) + else: + log.info('Updated Cvent Contact ID: {contact_id}') + try_request = False + + # log.info('Updated Cvent Contact ID: {contact_id}') + # response_data = resp.json() + # log.debug(json.dumps(response_data, indent=2, default=str)) if 'message' in response_data and response_data['message'] == 'Too Many Requests': return False return response_data -# ### END ### API Cvent Methods ### modify_contact_id() ### +# ### END ### API External Cvent Methods ### modify_contact_id() ### -# ### BEGIN ### API External Cvent ### create_update_aether_person() ### +# ### BEGIN ### API External Cvent Methods ### create_update_aether_person() ### # Updated 2022-02-02 @logger_reset def create_update_aether_person( @@ -530,8 +669,8 @@ def create_update_aether_person( custom_field_id = '609ab766-7d79-4a9d-a72c-f126412659ee' # IDAA Cvent External ID UUID custom_field_value = [person_external_id] - log.warning('Sleeping for .75 second to avoid Cvent rate limit...') - time.sleep(.75) + log.warning('Sleeping for .5 seconds to avoid Cvent rate limit...') + time.sleep(.5) contact_mod_result = modify_contact_id( contact_id = cvent_contact_id, # This is the Cvent Contact UUID for a person @@ -617,4 +756,4 @@ def create_update_aether_person( # log.debug(person_obj) return person_id # True -# ### END ### API External Cvent ### create_update_aether_person() ### +# ### END ### API External Cvent Methods ### create_update_aether_person() ### diff --git a/app/methods/user_methods.py b/app/methods/user_methods.py index 6a8e9bb..9cec531 100644 --- a/app/methods/user_methods.py +++ b/app/methods/user_methods.py @@ -181,7 +181,7 @@ def update_user_obj( fail_any: bool = True, # Fail if any thing goes wrong for sub objects return_dict: bool = False, ) -> bool|int: - log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # ### SECTION ### Secondary data validation @@ -197,17 +197,19 @@ def update_user_obj( log.debug(type(user_dict_obj)) if isinstance(user_dict_obj, dict): user_dict = user_dict_obj + user_dict['id'] = user_id try: user_obj = User_Base(**user_dict) - log.debug(user_obj) except ValidationError as e: log.error(e.json()) return False else: user_obj = user_dict_obj + user_obj.id = user_id + log.debug(user_obj) - # IMPORTANT NOTE: Need to be extra careful if allowing an update to password, super, or manager. Maybe other fields? - user_dict = user_obj.dict(by_alias=False, exclude_unset=True, exclude={'password', 'super', 'manager', 'contact', 'organization', 'person', 'created_on', 'updated_on'}) + # IMPORTANT NOTE: Need to be extra careful if allowing an update to password, super, or manager. Maybe other fields? + user_dict = user_obj.dict(by_alias=False, exclude_unset=True, exclude={'password', 'super', 'manager', 'contact', 'organization', 'person', 'created_on', 'updated_on'}) # log.debug(type(user_dict_obj)) @@ -309,15 +311,17 @@ def update_user_obj( user_obj.id = user_id # Is this needed? user_dict['id'] = user_id - if user_obj.new_password: - log.debug(user_obj.new_password) - else: - user_obj.new_password = secrets.token_urlsafe(default_num_bytes) - hash_string = secure_hash_string(string=user_obj.new_password) - user_obj.password = hash_string - user_dict['password'] = hash_string - + # if user_obj.new_password: log.debug(user_obj.new_password) + # if user_obj.password: + log.debug(user_obj.password) + # else: + # user_obj.new_password = secrets.token_urlsafe(default_num_bytes) + # hash_string = secure_hash_string(string=user_obj.new_password) + # user_obj.password = hash_string + # user_dict['password'] = hash_string + + # log.debug(user_obj.new_password) # Look for a person_id in the user_obj if person_id: pass diff --git a/app/routers/e_cvent.py b/app/routers/e_cvent.py index 19c4572..1863d33 100644 --- a/app/routers/e_cvent.py +++ b/app/routers/e_cvent.py @@ -9,7 +9,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template -from app.methods.e_cvent_methods import create_update_aether_person, get_access_token, get_contact_custom_field_list, get_contact_list, get_contact_id, get_recent_contact_list, modify_contact_id +from app.methods.e_cvent_methods import create_update_aether_person, get_access_token, get_contact_custom_field_list, get_contact_list, get_contact_id, get_group_contact_list, get_recent_contact_list, modify_contact_id from app.methods.person_methods import create_person_kiss, get_person_rec_list, get_person_rec_w_external_id, load_person_obj, update_person_kiss from app.methods.membership_person_methods import create_membership_person_obj, update_membership_person_obj @@ -26,10 +26,12 @@ router = APIRouter() @router.get('/person/process_recent_changes', response_model=Resp_Body_Base) async def process_recent_changes( from_created_on: datetime.datetime = datetime.datetime.utcnow() + datetime.timedelta(minutes=-60), # for Cvent "created" - to_created_on: datetime.datetime = datetime.datetime.utcnow(), # for Cvent "created" + to_created_on: datetime.datetime = None, # for Cvent "created" from_updated_on: datetime.datetime = datetime.datetime.utcnow() + datetime.timedelta(minutes=-60), # for Cvent "lastModified" to_updated_on: datetime.datetime = None, # for Cvent "lastModified" # type: str = 'created', # created, updated, created_updated + get_only: str = None, # created, updated/modified + contact_group_id: str = None, return_detail: bool = False, @@ -45,16 +47,26 @@ async def process_recent_changes( to_created_on = to_created_on, from_updated_on = from_updated_on, to_updated_on = to_updated_on, + get_only = 'created', ): log.debug(cvent_contact_list_result) cvent_contact_list = cvent_contact_list_result log.info(f'Found {len(cvent_contact_list)} Cvent contacts') + elif cvent_contact_list_result is None: + log.warning(f'No Cvent Contact results') + return mk_resp(data=None, status_code=404, response=commons.response) # Not Found else: log.warning(f'Something went wrong while trying to get recently created or updated person in Cvent. From Datetime: {from_created_on}') return mk_resp(data=None, status_code=400, response=commons.response) # Bad Request - # cvent_contact_list = cvent_contact_list_result.get('data') - # cvent_contact_list = cvent_contact_list_result + # contact_group_id = '60ef4838-5340-4e2a-9d0a-b4ef310eaa69' + if contact_group_id: + if cvent_group_contact_list_result := get_group_contact_list( + contact_group_id = contact_group_id, + ): + log.debug(cvent_group_contact_list_result) + cvent_group_contact_list = cvent_group_contact_list_result + log.info(f'Found {len(cvent_group_contact_list)} Cvent contacts in the group') # return mk_resp(data=cvent_contact_list, status_message=f'Checked for recent changes in Cvent. Found {len(cvent_contact_list)} Cvent contacts', response=commons.response) @@ -75,8 +87,8 @@ async def process_recent_changes( else: log.info(f'Something went wrong while trying to create or update the person in Aether based on Cvent data. Cvent (Person) Contact ID: {cvent_person_contact_id}') - log.warning('Sleeping for .5 second to avoid Cvent rate limit...') - time.sleep(.5) + # log.warning('Sleeping for .1 seconds to avoid Cvent rate limit...') + # time.sleep(.1) if return_detail: return mk_resp(data=cvent_contact_list_result, status_message=f'Checked for recent changes in Cvent. Created/Updated {len(cvent_contact_list)} Cvent contacts', response=commons.response)