import datetime, json, os, pprint, pytz, random, requests, shutil, string, time from typing import Dict, List, Optional, Set, Union from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator from app.config import settings from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update from app.lib_general import log, logging, logger_reset from app.methods.event_file_methods import create_event_file_obj from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, save_file_to_hosted_file from app.models.hosted_file_models import Hosted_File_Base from app.models.event_file_models import Event_File_Base api = {} # api['base_url'] = 'https://aapor.confex.com/aapor/2023/meetingapi.cgi/[object]/[id]' api['base_url'] = 'https://aapor.confex.com/aapor/2023/meetingapi.cgi' api['headers'] = { 'Content-Type': 'application/json;charset=UTF-8' } api['username'] = None api['password'] = None # ### BEGIN ### API External Confex Methods ### get_event_session_list() ### # Updated 2023-04-11 @logger_reset def get_event_session_list( ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # if result := authenticate(): # log.debug(result) # else: # return False endpoint = '/Session' uri = api['base_url']+endpoint params = {} confex_session_list = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') # log.debug(f'Encoding: {resp.encoding}') # log.debug('Text:') # log.debug(resp.text) # log.debug(resp.json()) if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_session_list_raw = resp.json() # .get('data').get('dataList')[0] # log.debug(confex_session_list_raw) confex_session_list = confex_session_list_raw log.debug(confex_session_list) try_request = False elif resp.status_code == 404: log.info('No results returned (status 404)') try_request = False confex_session_list = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_session_list = False else: log.info('Not trying again') try_request = False confex_session_list = False log.warning('Something may have gone wrong during the request.') # log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.') # api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired. return confex_session_list # ### END ### API External Confex Methods ### get_event_session_list() ### # ### BEGIN ### API External Confex Methods ### get_event_session_detail() ### # Updated 2023-04-11 @logger_reset def get_event_session_detail( confex_session_id: str, # actually an auto number ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # if result := authenticate(): # log.debug(result) # else: # return False endpoint = f'/Session/{confex_session_id}' uri = api['base_url']+endpoint params = {} confex_session_detail = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') # log.debug(f'Encoding: {resp.encoding}') # log.debug('Text:') # log.debug(resp.text) # log.debug(resp.json()) if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_session_detail_raw = resp.json() # .get('data').get('dataList')[0] # log.debug(confex_session_detail_raw) confex_session_detail = confex_session_detail_raw log.debug(confex_session_detail) try_request = False elif resp.status_code == 404: log.info('No results returned (status 404)') try_request = False confex_session_detail = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_session_detail = False else: log.info('Not trying again') try_request = False confex_session_detail = False log.warning('Something may have gone wrong during the request.') # log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.') # api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired. return confex_session_detail # ### END ### API External Confex Methods ### get_event_session_detail() ### # ### BEGIN ### API External Confex Methods ### get_event_presentation_detail() ### # Updated 2023-04-11 @logger_reset def get_event_presentation_detail( confex_session_id: str, confex_presentation_id: str, # similar to 'Paper/99999' ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # if result := authenticate(): # log.debug(result) # else: # return False # endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}' endpoint = f'/{confex_presentation_id}' uri = api['base_url']+endpoint params = {} confex_presentation_detail = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') # log.debug(f'Encoding: {resp.encoding}') # log.debug('Text:') # log.debug(resp.text) # log.debug(resp.json()) if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_presentation_detail_raw = resp.json() # .get('data').get('dataList')[0] # log.debug(confex_presentation_detail_raw) confex_presentation_detail = confex_presentation_detail_raw log.debug(confex_presentation_detail) try_request = False elif resp.status_code == 404: log.warning('No results returned (status 404)') try_request = False confex_presentation_detail = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_presentation_detail = False else: log.warning('Not trying again') try_request = False confex_presentation_detail = False log.warning('Something may have gone wrong during the request.') # log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.') # api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired. return confex_presentation_detail # ### END ### API External Confex Methods ### get_event_presentation_detail() ### # ### BEGIN ### API External Confex Methods ### get_event_presenter_detail() ### # Updated 2023-04-11 @logger_reset def get_event_presenter_detail( confex_session_id: str, confex_presentation_id: str, # similar to 'Paper/99999' confex_presenter_id: str, # similar to 'Person/99999' ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # if result := authenticate(): # log.debug(result) # else: # return False endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}/{confex_presenter_id}' uri = api['base_url']+endpoint params = {} confex_presenter_detail = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') # log.debug(f'Encoding: {resp.encoding}') # log.debug('Text:') # log.debug(resp.text) # log.debug(resp.json()) if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_presenter_detail_raw = resp.json() # .get('data').get('dataList')[0] # log.debug(confex_presenter_detail_raw) confex_presenter_detail = confex_presenter_detail_raw log.debug(confex_presenter_detail) try_request = False elif resp.status_code == 404: log.info('No results returned (status 404)') try_request = False confex_presenter_detail = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_presenter_detail = False else: log.warning('Not trying again') try_request = False confex_presenter_detail = False log.warning('Something may have gone wrong during the request.') # log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.') # api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired. return confex_presenter_detail # ### END ### API External Confex Methods ### get_event_presenter_detail() ### # ### BEGIN ### API External Confex Methods ### get_event_file_detail() ### # Updated 2023-04-30 @logger_reset def get_event_file_detail( confex_file_id: str, # similar to 'FileMap/Paper1928_Presentation2' ): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) endpoint = f'/{confex_file_id}' uri = api['base_url']+endpoint params = {} confex_file_detail = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_file_detail_raw = resp.json() # log.debug(confex_file_detail_raw) confex_file_detail = confex_file_detail_raw log.debug(confex_file_detail) try_request = False elif resp.status_code == 404: log.warning('No results returned (status 404)') try_request = False confex_file_detail = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_file_detail = False else: log.warning('Not trying again') try_request = False confex_file_detail = False log.warning('Something may have gone wrong during the request.') return confex_file_detail # ### END ### API External Confex Methods ### get_event_file_detail() ### # ### BEGIN ### API External Confex Methods ### get_event_slot_data() ### # Updated 2023-05-01 @logger_reset def get_event_slot_data( confex_slot_id: str, # similar to 'SlotData/Session1110_Slot119' ): log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) endpoint = f'/{confex_slot_id}' uri = api['base_url']+endpoint params = {} confex_slot_data = None try_request = True max_tries = 5 try_count = 0 while try_request and try_count <= max_tries: try_count = try_count + 1 resp = requests.get(url=uri, params=params, headers=api['headers']) log.debug(f'Status Code: {resp.status_code}') log.debug(f'Headers: {resp.headers}') if resp.status_code == 200: log.info('Status 200') log.debug(resp.json()) confex_slot_data_raw = resp.json() confex_slot_data = confex_slot_data_raw log.debug(confex_slot_data) try_request = False elif resp.status_code == 404: log.warning('No results returned (status 404)') try_request = False confex_slot_data = None elif resp.status_code == 429: log.warning('Hit rate limit. Sleeping for .1 seconds...') time.sleep(.1) try_request = True confex_slot_data = False else: log.warning('Not trying again') try_request = False confex_slot_data = False log.warning('Something may have gone wrong during the request.') return confex_slot_data # ### END ### API External Confex Methods ### get_event_slot_data() ### # Updated 2023-05-01 @logger_reset async def get_event_file_save_local( url: str, confex_file_info: dict, account_id: int, link_to_type: str, link_to_id: int, event_id: int = None, event_location_id: int = None, event_presentation_id: int = None, event_presenter_id: int = None, event_session_id: int = None, event_track_id: int = None, ) -> None|bool|dict: log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) # *** Part 1: *** First download the file to tmp and then save the hashed file to hosted_files directory. hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root'] log.info(f'Hosted Tmp Path: {hosted_tmp_path}') log.debug(shutil.disk_usage(hosted_tmp_path)) hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex', 'confex_temp.file') # hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex_temp.file') # return True # response = requests.get(url) response = requests.get(url, stream=True) # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL # log.debug(response) # log.debug(dir(response)) # file_obj = open(local_path, 'wb') with open(hosted_tmp_path_w_filename, 'wb') as f: for chunk in response.iter_content(chunk_size=1024): if chunk: f.write(chunk) # *** Part 2: *** Save the hashed file to hosted_files directory. file_info = await save_file_to_hosted_file( file_path = hosted_tmp_path_w_filename, filename = confex_file_info.get('filename'), extension = confex_file_info.get('extension'), account_id = account_id, link_to_type = link_to_type, link_to_id = link_to_id, ) # *** Part 3: *** Save information to database in hosted_file table (hosted_file_link table will be updated by an event_file table trigger) if file_info.get('saved'): # NOTE: Just in case look up in DB based on hash log.info('Look up in DB based on hash...') if hosted_file_sel_result := sql_select( table_name = 'hosted_file', field_name = 'hash_sha256', field_value = file_info['hash_sha256'], ): log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') # Got existing host_file object_entry! # Odd... the hash was found in the database, but the file had to be copied again. # If this happens then the file on the host server was probably deleted at some point. hosted_file_id = hosted_file_sel_result.get('id', None) hosted_file_id_random = hosted_file_sel_result.get('id_random', None) hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: # This is normal since the file was not found on the host server and not found in the DB. # Create a new host_file object entry and new host_file.id_random. file_info['account_id'] = account_id # file_info['account_id_random'] = account_id_random hosted_file_obj = Hosted_File_Base(**file_info) if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): hosted_file_id = hosted_file_obj_result hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: log.warning('For some reason a host_file object entry could not be created.') hosted_file_id = None hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member log.debug(hosted_file_obj_result) log.debug(hosted_file_sel_result) else: return False # *** Part 4: *** Save information to database in event_file (will trigger an update to hosted_file_link) event_file_data = {} event_file_data['hosted_file_id'] = hosted_file_id # event_file_data['hosted_file_id_random'] = hosted_file_id_random event_file_data['for_type'] = link_to_type event_file_data['for_id'] = link_to_id if event_id: event_file_data['event_id'] = event_id if event_location_id: event_file_data['event_location_id'] = event_location_id if event_presentation_id: event_file_data['event_presentation_id'] = event_presentation_id if event_presenter_id: event_file_data['event_presenter_id'] = event_presenter_id if event_session_id: event_file_data['event_session_id'] = event_session_id if event_track_id: event_file_data['event_track_id'] = event_track_id event_file_data['filename'] = file_info.get('filename') event_file_data['extension'] = file_info.get('extension') # event_file_data['open_in_os'] = event_file_obj.open_in_os # event_file_data['internal_use'] = event_file_obj.internal_use # event_file_data['public_use'] = hosted_file_obj.public_use # event_file_data['lu_file_purpose_id'] = hosted_file_obj.lu_file_purpose_id # event_file_data['file_purpose'] = hosted_file_obj.file_purpose # event_file_data['public'] = hosted_file_obj.public # event_file_data['hide'] = hosted_file_obj.hide event_file_data['enable'] = True # hosted_file_obj.enable # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(event_file_data) try: event_file_obj = Event_File_Base(**event_file_data) except ValidationError as e: log.error(e.json()) return False log.debug(event_file_obj) create_event_file_obj_result = create_event_file_obj(event_file_obj_new=event_file_obj) log.debug(create_event_file_obj_result) return file_info