559 lines
20 KiB
Python
559 lines
20 KiB
Python
import datetime, json, os, pprint, pytz, random, requests, shutil, string, time
|
|
|
|
from typing import Dict, List, Optional, Set, Union
|
|
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
|
|
|
from app.config import settings
|
|
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
|
from app.lib_general import log, logging, logger_reset
|
|
|
|
from app.methods.event_file_methods import create_event_file_obj
|
|
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, save_file_to_hosted_file
|
|
|
|
from app.models.hosted_file_models import Hosted_File_Base
|
|
from app.models.event_file_models import Event_File_Base
|
|
|
|
|
|
api = {}
|
|
# api['base_url'] = 'https://aapor.confex.com/aapor/2023/meetingapi.cgi/[object]/[id]'
|
|
api['base_url'] = 'https://aapor.confex.com/aapor/2023/meetingapi.cgi'
|
|
api['headers'] = { 'Content-Type': 'application/json;charset=UTF-8' }
|
|
api['username'] = None
|
|
api['password'] = None
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_session_list() ###
|
|
# Updated 2023-04-11
|
|
@logger_reset
|
|
def get_event_session_list(
|
|
):
|
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
# if result := authenticate():
|
|
# log.debug(result)
|
|
# else:
|
|
# return False
|
|
|
|
endpoint = '/Session'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_session_list = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
# log.debug(f'Encoding: {resp.encoding}')
|
|
# log.debug('Text:')
|
|
# log.debug(resp.text)
|
|
# log.debug(resp.json())
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_session_list_raw = resp.json() # .get('data').get('dataList')[0]
|
|
# log.debug(confex_session_list_raw)
|
|
|
|
confex_session_list = confex_session_list_raw
|
|
log.debug(confex_session_list)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.info('No results returned (status 404)')
|
|
try_request = False
|
|
confex_session_list = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_session_list = False
|
|
else:
|
|
log.info('Not trying again')
|
|
try_request = False
|
|
confex_session_list = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.')
|
|
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
|
|
|
|
return confex_session_list
|
|
# ### END ### API External Confex Methods ### get_event_session_list() ###
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_session_detail() ###
|
|
# Updated 2023-04-11
|
|
@logger_reset
|
|
def get_event_session_detail(
|
|
confex_session_id: str, # actually an auto number
|
|
):
|
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
# if result := authenticate():
|
|
# log.debug(result)
|
|
# else:
|
|
# return False
|
|
|
|
endpoint = f'/Session/{confex_session_id}'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_session_detail = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
# log.debug(f'Encoding: {resp.encoding}')
|
|
# log.debug('Text:')
|
|
# log.debug(resp.text)
|
|
# log.debug(resp.json())
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_session_detail_raw = resp.json() # .get('data').get('dataList')[0]
|
|
# log.debug(confex_session_detail_raw)
|
|
|
|
confex_session_detail = confex_session_detail_raw
|
|
log.debug(confex_session_detail)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.info('No results returned (status 404)')
|
|
try_request = False
|
|
confex_session_detail = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_session_detail = False
|
|
else:
|
|
log.info('Not trying again')
|
|
try_request = False
|
|
confex_session_detail = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.')
|
|
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
|
|
|
|
return confex_session_detail
|
|
# ### END ### API External Confex Methods ### get_event_session_detail() ###
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_presentation_detail() ###
|
|
# Updated 2023-04-11
|
|
@logger_reset
|
|
def get_event_presentation_detail(
|
|
confex_session_id: str,
|
|
confex_presentation_id: str, # similar to 'Paper/99999'
|
|
):
|
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
# if result := authenticate():
|
|
# log.debug(result)
|
|
# else:
|
|
# return False
|
|
|
|
# endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}'
|
|
endpoint = f'/{confex_presentation_id}'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_presentation_detail = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
# log.debug(f'Encoding: {resp.encoding}')
|
|
# log.debug('Text:')
|
|
# log.debug(resp.text)
|
|
# log.debug(resp.json())
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_presentation_detail_raw = resp.json() # .get('data').get('dataList')[0]
|
|
# log.debug(confex_presentation_detail_raw)
|
|
|
|
confex_presentation_detail = confex_presentation_detail_raw
|
|
log.debug(confex_presentation_detail)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.warning('No results returned (status 404)')
|
|
try_request = False
|
|
confex_presentation_detail = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_presentation_detail = False
|
|
else:
|
|
log.warning('Not trying again')
|
|
try_request = False
|
|
confex_presentation_detail = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.')
|
|
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
|
|
|
|
return confex_presentation_detail
|
|
# ### END ### API External Confex Methods ### get_event_presentation_detail() ###
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_presenter_detail() ###
|
|
# Updated 2023-04-11
|
|
@logger_reset
|
|
def get_event_presenter_detail(
|
|
confex_session_id: str,
|
|
confex_presentation_id: str, # similar to 'Paper/99999'
|
|
confex_presenter_id: str, # similar to 'Person/99999'
|
|
):
|
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
# if result := authenticate():
|
|
# log.debug(result)
|
|
# else:
|
|
# return False
|
|
|
|
endpoint = f'/Session/{confex_session_id}/{confex_presentation_id}/{confex_presenter_id}'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_presenter_detail = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
# log.debug(f'Encoding: {resp.encoding}')
|
|
# log.debug('Text:')
|
|
# log.debug(resp.text)
|
|
# log.debug(resp.json())
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_presenter_detail_raw = resp.json() # .get('data').get('dataList')[0]
|
|
# log.debug(confex_presenter_detail_raw)
|
|
|
|
confex_presenter_detail = confex_presenter_detail_raw
|
|
log.debug(confex_presenter_detail)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.info('No results returned (status 404)')
|
|
try_request = False
|
|
confex_presenter_detail = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_presenter_detail = False
|
|
else:
|
|
log.warning('Not trying again')
|
|
try_request = False
|
|
confex_presenter_detail = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Confex on the next request.')
|
|
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
|
|
|
|
return confex_presenter_detail
|
|
# ### END ### API External Confex Methods ### get_event_presenter_detail() ###
|
|
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_file_detail() ###
|
|
# Updated 2023-04-30
|
|
@logger_reset
|
|
def get_event_file_detail(
|
|
confex_file_id: str, # similar to 'FileMap/Paper1928_Presentation2'
|
|
):
|
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
endpoint = f'/{confex_file_id}'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_file_detail = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_file_detail_raw = resp.json()
|
|
# log.debug(confex_file_detail_raw)
|
|
|
|
confex_file_detail = confex_file_detail_raw
|
|
log.debug(confex_file_detail)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.warning('No results returned (status 404)')
|
|
try_request = False
|
|
confex_file_detail = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_file_detail = False
|
|
else:
|
|
log.warning('Not trying again')
|
|
try_request = False
|
|
confex_file_detail = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
return confex_file_detail
|
|
# ### END ### API External Confex Methods ### get_event_file_detail() ###
|
|
|
|
|
|
# ### BEGIN ### API External Confex Methods ### get_event_slot_data() ###
|
|
# Updated 2023-05-01
|
|
@logger_reset
|
|
def get_event_slot_data(
|
|
confex_slot_id: str, # similar to 'SlotData/Session1110_Slot119'
|
|
):
|
|
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
endpoint = f'/{confex_slot_id}'
|
|
uri = api['base_url']+endpoint
|
|
params = {}
|
|
|
|
confex_slot_data = None
|
|
|
|
try_request = True
|
|
max_tries = 5
|
|
try_count = 0
|
|
while try_request and try_count <= max_tries:
|
|
try_count = try_count + 1
|
|
|
|
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
|
|
|
log.debug(f'Status Code: {resp.status_code}')
|
|
log.debug(f'Headers: {resp.headers}')
|
|
|
|
if resp.status_code == 200:
|
|
log.info('Status 200')
|
|
log.debug(resp.json())
|
|
|
|
confex_slot_data_raw = resp.json()
|
|
|
|
confex_slot_data = confex_slot_data_raw
|
|
log.debug(confex_slot_data)
|
|
|
|
try_request = False
|
|
elif resp.status_code == 404:
|
|
log.warning('No results returned (status 404)')
|
|
try_request = False
|
|
confex_slot_data = None
|
|
elif resp.status_code == 429:
|
|
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
|
time.sleep(.1)
|
|
try_request = True
|
|
confex_slot_data = False
|
|
else:
|
|
log.warning('Not trying again')
|
|
try_request = False
|
|
confex_slot_data = False
|
|
|
|
log.warning('Something may have gone wrong during the request.')
|
|
|
|
return confex_slot_data
|
|
# ### END ### API External Confex Methods ### get_event_slot_data() ###
|
|
|
|
|
|
# Updated 2023-05-01
|
|
@logger_reset
|
|
async def get_event_file_save_local(
|
|
url: str,
|
|
confex_file_info: dict,
|
|
account_id: int,
|
|
link_to_type: str,
|
|
link_to_id: int,
|
|
event_id: int = None,
|
|
event_location_id: int = None,
|
|
event_presentation_id: int = None,
|
|
event_presenter_id: int = None,
|
|
event_session_id: int = None,
|
|
event_track_id: int = None,
|
|
) -> None|bool|dict:
|
|
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(locals())
|
|
|
|
|
|
# *** Part 1: *** First download the file to tmp and then save the hashed file to hosted_files directory.
|
|
|
|
hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root']
|
|
log.info(f'Hosted Tmp Path: {hosted_tmp_path}')
|
|
log.debug(shutil.disk_usage(hosted_tmp_path))
|
|
|
|
hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex', 'confex_temp.file')
|
|
# hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex_temp.file')
|
|
|
|
# return True
|
|
|
|
# response = requests.get(url)
|
|
response = requests.get(url, stream=True)
|
|
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
# log.debug(response)
|
|
# log.debug(dir(response))
|
|
|
|
# file_obj = open(local_path, 'wb')
|
|
|
|
with open(hosted_tmp_path_w_filename, 'wb') as f:
|
|
for chunk in response.iter_content(chunk_size=1024):
|
|
if chunk:
|
|
f.write(chunk)
|
|
|
|
# *** Part 2: *** Save the hashed file to hosted_files directory.
|
|
|
|
file_info = await save_file_to_hosted_file(
|
|
file_path = hosted_tmp_path_w_filename,
|
|
filename = confex_file_info.get('filename'),
|
|
extension = confex_file_info.get('extension'),
|
|
account_id = account_id,
|
|
link_to_type = link_to_type,
|
|
link_to_id = link_to_id,
|
|
)
|
|
|
|
# *** Part 3: *** Save information to database in hosted_file table (hosted_file_link table will be updated by an event_file table trigger)
|
|
|
|
if file_info.get('saved'):
|
|
# NOTE: Just in case look up in DB based on hash
|
|
log.info('Look up in DB based on hash...')
|
|
if hosted_file_sel_result := sql_select(
|
|
table_name = 'hosted_file',
|
|
field_name = 'hash_sha256',
|
|
field_value = file_info['hash_sha256'],
|
|
):
|
|
log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!')
|
|
# Got existing host_file object_entry!
|
|
# Odd... the hash was found in the database, but the file had to be copied again.
|
|
# If this happens then the file on the host server was probably deleted at some point.
|
|
hosted_file_id = hosted_file_sel_result.get('id', None)
|
|
hosted_file_id_random = hosted_file_sel_result.get('id_random', None)
|
|
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
|
else:
|
|
# This is normal since the file was not found on the host server and not found in the DB.
|
|
# Create a new host_file object entry and new host_file.id_random.
|
|
file_info['account_id'] = account_id
|
|
# file_info['account_id_random'] = account_id_random
|
|
hosted_file_obj = Hosted_File_Base(**file_info)
|
|
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
|
|
hosted_file_id = hosted_file_obj_result
|
|
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
|
else:
|
|
log.warning('For some reason a host_file object entry could not be created.')
|
|
hosted_file_id = None
|
|
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
|
|
log.debug(hosted_file_obj_result)
|
|
log.debug(hosted_file_sel_result)
|
|
else: return False
|
|
|
|
|
|
# *** Part 4: *** Save information to database in event_file (will trigger an update to hosted_file_link)
|
|
|
|
event_file_data = {}
|
|
event_file_data['hosted_file_id'] = hosted_file_id
|
|
# event_file_data['hosted_file_id_random'] = hosted_file_id_random
|
|
|
|
event_file_data['for_type'] = link_to_type
|
|
event_file_data['for_id'] = link_to_id
|
|
|
|
if event_id:
|
|
event_file_data['event_id'] = event_id
|
|
if event_location_id:
|
|
event_file_data['event_location_id'] = event_location_id
|
|
if event_presentation_id:
|
|
event_file_data['event_presentation_id'] = event_presentation_id
|
|
if event_presenter_id:
|
|
event_file_data['event_presenter_id'] = event_presenter_id
|
|
if event_session_id:
|
|
event_file_data['event_session_id'] = event_session_id
|
|
if event_track_id:
|
|
event_file_data['event_track_id'] = event_track_id
|
|
|
|
event_file_data['filename'] = file_info.get('filename')
|
|
event_file_data['extension'] = file_info.get('extension')
|
|
|
|
# event_file_data['open_in_os'] = event_file_obj.open_in_os
|
|
# event_file_data['internal_use'] = event_file_obj.internal_use
|
|
|
|
# event_file_data['public_use'] = hosted_file_obj.public_use
|
|
|
|
# event_file_data['lu_file_purpose_id'] = hosted_file_obj.lu_file_purpose_id
|
|
# event_file_data['file_purpose'] = hosted_file_obj.file_purpose
|
|
|
|
# event_file_data['public'] = hosted_file_obj.public
|
|
# event_file_data['hide'] = hosted_file_obj.hide
|
|
event_file_data['enable'] = True # hosted_file_obj.enable
|
|
|
|
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
|
log.debug(event_file_data)
|
|
|
|
try:
|
|
event_file_obj = Event_File_Base(**event_file_data)
|
|
except ValidationError as e:
|
|
log.error(e.json())
|
|
return False
|
|
log.debug(event_file_obj)
|
|
|
|
create_event_file_obj_result = create_event_file_obj(event_file_obj_new=event_file_obj)
|
|
log.debug(create_event_file_obj_result)
|
|
|
|
|
|
return file_info
|