Prep for AAPOR with Confex and NCSD STD. General clean up.
This commit is contained in:
Scott Idem
@@ -1,11 +1,18 @@
|
||||
import datetime, json, pprint, pytz, random, requests, string, time
|
||||
import datetime, json, os, pprint, pytz, random, requests, shutil, string, time
|
||||
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.config import settings
|
||||
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
||||
from app.lib_general import log, logging, logger_reset
|
||||
|
||||
from app.methods.event_file_methods import create_event_file_obj
|
||||
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, save_file_to_hosted_file
|
||||
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
from app.models.event_file_models import Event_File_Base
|
||||
|
||||
|
||||
api = {}
|
||||
# api['base_url'] = 'https://aapor.confex.com/aapor/2023/meetingapi.cgi/[object]/[id]'
|
||||
@@ -20,7 +27,7 @@ api['password'] = None
|
||||
@logger_reset
|
||||
def get_event_session_list(
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if result := authenticate():
|
||||
@@ -89,7 +96,7 @@ def get_event_session_list(
|
||||
def get_event_session_detail(
|
||||
confex_session_id: str, # actually an auto number
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if result := authenticate():
|
||||
@@ -231,7 +238,7 @@ def get_event_presenter_detail(
|
||||
confex_presentation_id: str, # similar to 'Paper/99999'
|
||||
confex_presenter_id: str, # similar to 'Person/99999'
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if result := authenticate():
|
||||
@@ -281,7 +288,7 @@ def get_event_presenter_detail(
|
||||
try_request = True
|
||||
confex_presenter_detail = False
|
||||
else:
|
||||
log.info('Not trying again')
|
||||
log.warning('Not trying again')
|
||||
try_request = False
|
||||
confex_presenter_detail = False
|
||||
|
||||
@@ -301,14 +308,9 @@ def get_event_presenter_detail(
|
||||
def get_event_file_detail(
|
||||
confex_file_id: str, # similar to 'FileMap/Paper1928_Presentation2'
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if result := authenticate():
|
||||
# log.debug(result)
|
||||
# else:
|
||||
# return False
|
||||
|
||||
endpoint = f'/{confex_file_id}'
|
||||
uri = api['base_url']+endpoint
|
||||
params = {}
|
||||
@@ -325,16 +327,12 @@ def get_event_file_detail(
|
||||
|
||||
log.debug(f'Status Code: {resp.status_code}')
|
||||
log.debug(f'Headers: {resp.headers}')
|
||||
# log.debug(f'Encoding: {resp.encoding}')
|
||||
# log.debug('Text:')
|
||||
# log.debug(resp.text)
|
||||
# log.debug(resp.json())
|
||||
|
||||
if resp.status_code == 200:
|
||||
log.info('Status 200')
|
||||
log.debug(resp.json())
|
||||
|
||||
confex_file_detail_raw = resp.json() # .get('data').get('dataList')[0]
|
||||
confex_file_detail_raw = resp.json()
|
||||
# log.debug(confex_file_detail_raw)
|
||||
|
||||
confex_file_detail = confex_file_detail_raw
|
||||
@@ -357,8 +355,204 @@ def get_event_file_detail(
|
||||
|
||||
log.warning('Something may have gone wrong during the request.')
|
||||
|
||||
# log.warning('Something may have gone wrong. Setting the API app_user_token_datetime value to None to re-authenticate with Impexium on the next request.')
|
||||
# api['app_user_token_datetime'] = None # Resetting this just in case the App and or User token expired.
|
||||
|
||||
return confex_file_detail
|
||||
# ### END ### API External Confex Methods ### get_event_file_detail() ###
|
||||
# ### END ### API External Confex Methods ### get_event_file_detail() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API External Confex Methods ### get_event_slot_data() ###
|
||||
# Updated 2023-05-01
|
||||
@logger_reset
|
||||
def get_event_slot_data(
|
||||
confex_slot_id: str, # similar to 'SlotData/Session1110_Slot119'
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
endpoint = f'/{confex_slot_id}'
|
||||
uri = api['base_url']+endpoint
|
||||
params = {}
|
||||
|
||||
confex_slot_data = None
|
||||
|
||||
try_request = True
|
||||
max_tries = 5
|
||||
try_count = 0
|
||||
while try_request and try_count <= max_tries:
|
||||
try_count = try_count + 1
|
||||
|
||||
resp = requests.get(url=uri, params=params, headers=api['headers'])
|
||||
|
||||
log.debug(f'Status Code: {resp.status_code}')
|
||||
log.debug(f'Headers: {resp.headers}')
|
||||
|
||||
if resp.status_code == 200:
|
||||
log.info('Status 200')
|
||||
log.debug(resp.json())
|
||||
|
||||
confex_slot_data_raw = resp.json()
|
||||
|
||||
confex_slot_data = confex_slot_data_raw
|
||||
log.debug(confex_slot_data)
|
||||
|
||||
try_request = False
|
||||
elif resp.status_code == 404:
|
||||
log.warning('No results returned (status 404)')
|
||||
try_request = False
|
||||
confex_slot_data = None
|
||||
elif resp.status_code == 429:
|
||||
log.warning('Hit rate limit. Sleeping for .1 seconds...')
|
||||
time.sleep(.1)
|
||||
try_request = True
|
||||
confex_slot_data = False
|
||||
else:
|
||||
log.warning('Not trying again')
|
||||
try_request = False
|
||||
confex_slot_data = False
|
||||
|
||||
log.warning('Something may have gone wrong during the request.')
|
||||
|
||||
return confex_slot_data
|
||||
# ### END ### API External Confex Methods ### get_event_slot_data() ###
|
||||
|
||||
|
||||
# Updated 2023-05-01
|
||||
@logger_reset
|
||||
async def get_event_file_save_local(
|
||||
url: str,
|
||||
confex_file_info: dict,
|
||||
account_id: int,
|
||||
link_to_type: str,
|
||||
link_to_id: int,
|
||||
event_id: int = None,
|
||||
event_location_id: int = None,
|
||||
event_presentation_id: int = None,
|
||||
event_presenter_id: int = None,
|
||||
event_session_id: int = None,
|
||||
event_track_id: int = None,
|
||||
) -> None|bool|dict:
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
|
||||
# *** Part 1: *** First download the file to tmp and then save the hashed file to hosted_files directory.
|
||||
|
||||
hosted_tmp_path = settings.FILES_PATH['hosted_tmp_root']
|
||||
log.info(f'Hosted Tmp Path: {hosted_tmp_path}')
|
||||
log.debug(shutil.disk_usage(hosted_tmp_path))
|
||||
|
||||
hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex', 'confex_temp.file')
|
||||
# hosted_tmp_path_w_filename = os.path.join(hosted_tmp_path, 'confex_temp.file')
|
||||
|
||||
# return True
|
||||
|
||||
# response = requests.get(url)
|
||||
response = requests.get(url, stream=True)
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
# log.debug(response)
|
||||
# log.debug(dir(response))
|
||||
|
||||
# file_obj = open(local_path, 'wb')
|
||||
|
||||
with open(hosted_tmp_path_w_filename, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=1024):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
|
||||
# *** Part 2: *** Save the hashed file to hosted_files directory.
|
||||
|
||||
file_info = await save_file_to_hosted_file(
|
||||
file_path = hosted_tmp_path_w_filename,
|
||||
filename = confex_file_info.get('filename'),
|
||||
extension = confex_file_info.get('extension'),
|
||||
account_id = account_id,
|
||||
link_to_type = link_to_type,
|
||||
link_to_id = link_to_id,
|
||||
)
|
||||
|
||||
# *** Part 3: *** Save information to database in hosted_file table (hosted_file_link table will be updated by an event_file table trigger)
|
||||
|
||||
if file_info.get('saved'):
|
||||
# NOTE: Just in case look up in DB based on hash
|
||||
log.info('Look up in DB based on hash...')
|
||||
if hosted_file_sel_result := sql_select(
|
||||
table_name = 'hosted_file',
|
||||
field_name = 'hash_sha256',
|
||||
field_value = file_info['hash_sha256'],
|
||||
):
|
||||
log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!')
|
||||
# Got existing host_file object_entry!
|
||||
# Odd... the hash was found in the database, but the file had to be copied again.
|
||||
# If this happens then the file on the host server was probably deleted at some point.
|
||||
hosted_file_id = hosted_file_sel_result.get('id', None)
|
||||
hosted_file_id_random = hosted_file_sel_result.get('id_random', None)
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
else:
|
||||
# This is normal since the file was not found on the host server and not found in the DB.
|
||||
# Create a new host_file object entry and new host_file.id_random.
|
||||
file_info['account_id'] = account_id
|
||||
# file_info['account_id_random'] = account_id_random
|
||||
hosted_file_obj = Hosted_File_Base(**file_info)
|
||||
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
|
||||
hosted_file_id = hosted_file_obj_result
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
else:
|
||||
log.warning('For some reason a host_file object entry could not be created.')
|
||||
hosted_file_id = None
|
||||
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
|
||||
log.debug(hosted_file_obj_result)
|
||||
log.debug(hosted_file_sel_result)
|
||||
else: return False
|
||||
|
||||
|
||||
# *** Part 4: *** Save information to database in event_file (will trigger an update to hosted_file_link)
|
||||
|
||||
event_file_data = {}
|
||||
event_file_data['hosted_file_id'] = hosted_file_id
|
||||
# event_file_data['hosted_file_id_random'] = hosted_file_id_random
|
||||
|
||||
event_file_data['for_type'] = link_to_type
|
||||
event_file_data['for_id'] = link_to_id
|
||||
|
||||
if event_id:
|
||||
event_file_data['event_id'] = event_id
|
||||
if event_location_id:
|
||||
event_file_data['event_location_id'] = event_location_id
|
||||
if event_presentation_id:
|
||||
event_file_data['event_presentation_id'] = event_presentation_id
|
||||
if event_presenter_id:
|
||||
event_file_data['event_presenter_id'] = event_presenter_id
|
||||
if event_session_id:
|
||||
event_file_data['event_session_id'] = event_session_id
|
||||
if event_track_id:
|
||||
event_file_data['event_track_id'] = event_track_id
|
||||
|
||||
event_file_data['filename'] = file_info.get('filename')
|
||||
event_file_data['extension'] = file_info.get('extension')
|
||||
|
||||
# event_file_data['open_in_os'] = event_file_obj.open_in_os
|
||||
# event_file_data['internal_use'] = event_file_obj.internal_use
|
||||
|
||||
# event_file_data['public_use'] = hosted_file_obj.public_use
|
||||
|
||||
# event_file_data['lu_file_purpose_id'] = hosted_file_obj.lu_file_purpose_id
|
||||
# event_file_data['file_purpose'] = hosted_file_obj.file_purpose
|
||||
|
||||
# event_file_data['public'] = hosted_file_obj.public
|
||||
# event_file_data['hide'] = hosted_file_obj.hide
|
||||
event_file_data['enable'] = True # hosted_file_obj.enable
|
||||
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(event_file_data)
|
||||
|
||||
try:
|
||||
event_file_obj = Event_File_Base(**event_file_data)
|
||||
except ValidationError as e:
|
||||
log.error(e.json())
|
||||
return False
|
||||
log.debug(event_file_obj)
|
||||
|
||||
create_event_file_obj_result = create_event_file_obj(event_file_obj_new=event_file_obj)
|
||||
log.debug(create_event_file_obj_result)
|
||||
|
||||
|
||||
return file_info
|
||||
|
||||
@@ -328,7 +328,7 @@ def get_event_session_rec_list(
|
||||
approved: str = 'all', # approved, not_approved, all
|
||||
hidden: str = 'not_hidden', # hidden, not_hidden, all
|
||||
review: str = 'all', # ready, not_ready, all
|
||||
limit: int = 100,
|
||||
limit: int = 150,
|
||||
offset: int = 0,
|
||||
) -> list|bool:
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
@@ -441,7 +441,7 @@ def create_update_event_session_obj_v4(
|
||||
fail_any: bool = False, # Fail if any thing goes wrong for sub objects
|
||||
return_outline: bool = False,
|
||||
) -> int|bool:
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
log.info('Checking requirements...')
|
||||
|
||||
@@ -44,7 +44,7 @@ def load_hosted_file_obj(
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_hosted_file_link_list: bool = False,
|
||||
) -> Hosted_File_Base|dict|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
@@ -116,7 +116,7 @@ def lookup_file_hash(
|
||||
# ### END ### API Hosted File Methods ### lookup_file_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### get_file_object_hash() ###
|
||||
@logger_reset
|
||||
async def get_file_object_hash(file_object:File):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
@@ -137,25 +137,25 @@ async def get_file_object_hash(file_object:File):
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
|
||||
return file_hash
|
||||
# ### END ### API Hosted File Route ### get_file_object_hash() ###
|
||||
# ### END ### API Hosted File Methods ### get_file_object_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### guess_file_extension() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### guess_file_extension() ###
|
||||
def guess_file_extension(filename: str):
|
||||
return filename.rsplit('.', 1)[1].lower()
|
||||
# ### END ### API Hosted File Route ### guess_file_extension() ###
|
||||
# ### END ### API Hosted File Methods ### guess_file_extension() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### allowed_file_extension() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### allowed_file_extension() ###
|
||||
def allowed_file_extension(extension: str, extension_list: list):
|
||||
return extension.lower() in extension_list # app.config['ALLOWED_EXTENSIONS']
|
||||
# ### END ### API Hosted File Route ### allowed_file_extension() ###
|
||||
# ### END ### API Hosted File Methods ### allowed_file_extension() ###
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### save_file() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### save_file() ###
|
||||
# Updated 2022-08-09
|
||||
@logger_reset
|
||||
async def save_file(
|
||||
@@ -167,7 +167,7 @@ async def save_file(
|
||||
link_to_id_random: str = None,
|
||||
check_allowed_extension: bool = False,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
@@ -371,7 +371,134 @@ async def save_file(
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Route ### save_file() ###
|
||||
# ### END ### API Hosted File Methods ### save_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### save_file_to_hosted_file() ###
|
||||
# Updated 2022-08-09
|
||||
@logger_reset
|
||||
async def save_file_to_hosted_file(
|
||||
file_path: str,
|
||||
filename: str,
|
||||
extension: str,
|
||||
account_id: int,
|
||||
link_to_type: str,
|
||||
link_to_id: int,
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
log.debug(file_path)
|
||||
log.debug(f'Filename: {filename} Extension: {extension}')
|
||||
|
||||
file_obj = open(file_path, 'rb')
|
||||
|
||||
|
||||
file_info: dict = {}
|
||||
file_info['saved'] = None
|
||||
file_info['link_to_type'] = link_to_type
|
||||
file_info['link_to_id'] = link_to_id
|
||||
file_info['filename'] = filename
|
||||
file_info['extension'] = extension # guess_file_extension(filename=filename)
|
||||
|
||||
# if check_allowed_extension:
|
||||
# if allowed_file_extension(extension=file_info['extension'], extension_list=['jpg','png','webp']):
|
||||
# file_info['extension_allowed'] = True
|
||||
# else:
|
||||
# file_info['extension_allowed'] = False
|
||||
# file_info['saved'] = False
|
||||
# return file_info
|
||||
# else:
|
||||
# file_info['extension_allowed'] = None
|
||||
|
||||
# There is a difference between Content-Type and MIME type.
|
||||
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
|
||||
# file_info['content_type'] = file.content_type # might also include charset or other parameters
|
||||
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
|
||||
|
||||
file_obj.seek(0, os.SEEK_END)
|
||||
file_size = file_obj.tell()
|
||||
file_obj.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
log.debug(file_size)
|
||||
file_info['size'] = file_size
|
||||
|
||||
file_hash = await get_file_object_hash(file_obj)
|
||||
log.debug(file_hash)
|
||||
file_info['hash_sha256'] = file_hash
|
||||
|
||||
# 16384 bytes is the default
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
buffer_size = 524288
|
||||
|
||||
#f_src = open(file_src, 'rb')
|
||||
f_src = file_obj # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||
|
||||
file_hash_subdirectory = file_hash[0:2]
|
||||
subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory)
|
||||
log.debug(subdirectory_dest)
|
||||
pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True)
|
||||
file_info['subdirectory_path'] = file_hash_subdirectory
|
||||
|
||||
#file_dest = f'{hosted_files_path}{file.filename}'
|
||||
# file_dest = f'{hosted_files_path}{file_hash}.file'
|
||||
|
||||
file_dest = os.path.join(hosted_files_path, f'{file_hash}.file')
|
||||
file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file')
|
||||
|
||||
existing_file_check = pathlib.Path(file_dest)
|
||||
existing_file_check_subdir = pathlib.Path(file_dest_w_subdir)
|
||||
|
||||
log.debug(existing_file_check_subdir)
|
||||
# return file_info
|
||||
|
||||
|
||||
if existing_file_check_subdir.exists():
|
||||
log.warning('This file already exists at the destination with the subdirectory. Not re-saving.')
|
||||
file_info['already_exists'] = True
|
||||
file_info['already_exists_subdir'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.warning('This file does not already exist at the destination subdirectory.')
|
||||
file_info['already_exists'] = False
|
||||
file_info['already_exists_subdir'] = False
|
||||
try:
|
||||
log.info('Saving file to destination...')
|
||||
f_dest = open(file_dest_w_subdir, 'wb')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
|
||||
log.info(f'File saved to: {hosted_files_path}')
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
log.exception('***')
|
||||
log.exception(str(e))
|
||||
log.exception('^^^ exception ^^^')
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
return False
|
||||
log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}')
|
||||
log.info(f"Filename: {file_info['filename']}")
|
||||
log.info(f"Subdirectory Path: {file_info['subdirectory_path']}")
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(file_info)
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Methods ### save_file_to_hosted_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||
|
||||
@@ -16,7 +16,8 @@ from app.methods.event_presentation_methods import create_update_event_presentat
|
||||
from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4
|
||||
from app.methods.event_session_methods import create_update_event_session_obj_v4
|
||||
# from app.methods.event_person_methods import create_event_person_obj, create_update_event_person_obj_v4, get_event_person_rec_list, load_event_person_obj, update_event_person_obj, update_event_person_obj_v3
|
||||
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail
|
||||
from app.methods.e_confex_methods import get_event_session_list, get_event_session_detail, get_event_presentation_detail, get_event_presenter_detail, get_event_file_detail, get_event_file_save_local, get_event_slot_data
|
||||
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
|
||||
|
||||
from app.models.event_presentation_models import Event_Presentation_Base
|
||||
from app.models.event_presenter_models import Event_Presenter_Base
|
||||
@@ -40,7 +41,7 @@ async def import_event_session_list(
|
||||
|
||||
commons: Common_Route_Params = Depends(common_route_params),
|
||||
):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
log.warning('Starting Confex event session list import...')
|
||||
@@ -65,7 +66,7 @@ async def import_event_session_list(
|
||||
# BEGIN: Loop through the sessions pulled from Confex
|
||||
|
||||
loop_count = 0
|
||||
loop_limit = 200 # 200
|
||||
loop_limit = 220 # 200
|
||||
event_session_summary_li = []
|
||||
for event_session in event_session_li:
|
||||
if loop_count > loop_limit: break
|
||||
@@ -73,13 +74,84 @@ async def import_event_session_list(
|
||||
|
||||
confex_session_id = event_session.get('id')
|
||||
|
||||
if confex_session_id in ('1181'): # '1110', '1100'
|
||||
# log.setLevel(logging.DEBUG)
|
||||
log.warning(f'**************** FOUND {confex_session_id} ******************')
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
log.warning(f'*** FOUND {confex_session_id} ***')
|
||||
continue
|
||||
# if confex_session_id in ('1181', '1110', '1100', '1076'): # '1110', '1100'
|
||||
# # log.setLevel(logging.DEBUG)
|
||||
# log.warning(f'**************** FOUND {confex_session_id} ******************')
|
||||
# else:
|
||||
# log.setLevel(logging.INFO)
|
||||
# log.warning(f'*** FOUND {confex_session_id} ***')
|
||||
# continue
|
||||
|
||||
# *** Part 1: *** Deal with the event_location first
|
||||
# log.setLevel(logging.DEBUG)
|
||||
log.debug(event_session.get('ParentList_SlotData'))
|
||||
if event_session.get('ParentList_SlotData') and len(event_session.get('ParentList_SlotData')):
|
||||
confex_slot_id = event_session.get('ParentList_SlotData')[0]
|
||||
confex_slot_data = get_event_slot_data(confex_slot_id = confex_slot_id)
|
||||
log.debug(confex_slot_data)
|
||||
# event_location_external_id = confex_slot_id # This will not work because it is not unique per room.
|
||||
event_location_external_id = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
|
||||
event_location_code = str(confex_slot_data.get('Room_Floor'))+'/'+str(confex_slot_data.get('Room_Name')) # No better option
|
||||
event_location_name = confex_slot_data.get('Room_Name')
|
||||
|
||||
data = {}
|
||||
data['account_id'] = account_id
|
||||
data['event_id'] = event_id
|
||||
|
||||
# ### INSERT or UPDATE event_location
|
||||
# NOTE: In the future it may make more sense to pull this location part out and or add a checked ID array so we can avoid SQL queries.
|
||||
data['event_location_external_id'] = event_location_external_id
|
||||
|
||||
sql = f"""
|
||||
SELECT *
|
||||
FROM `v_event_location` AS `event_location`
|
||||
WHERE event_location.event_id = :event_id
|
||||
AND event_location.external_id = :event_location_external_id
|
||||
LIMIT 1;
|
||||
"""
|
||||
|
||||
event_location_data = {}
|
||||
event_location_data['code'] = event_location_code
|
||||
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
|
||||
if event_location_name:
|
||||
event_location_data['name'] = event_location_name
|
||||
else:
|
||||
event_location_data['name'] = 'Unknown'
|
||||
event_location_data['enable'] = True
|
||||
|
||||
event_location_id = None
|
||||
if event_location_rec_result := sql_select(data=data, sql=sql):
|
||||
# Pull out IDs and UPDATE existing event_location record
|
||||
event_location_rec = event_location_rec_result
|
||||
event_location_id = event_location_rec.get('event_location_id')
|
||||
event_location_id_random = event_location_rec.get('event_location_id_random')
|
||||
event_location_name = event_location_data.get('name')
|
||||
log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}')
|
||||
|
||||
event_location_data['id'] = event_location_id
|
||||
|
||||
log.debug(event_location_data)
|
||||
|
||||
if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'):
|
||||
log.debug(event_location_obj_up_result)
|
||||
else:
|
||||
log.warning(event_location_obj_up_result)
|
||||
pass # Probably nothing to update
|
||||
else:
|
||||
log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}')
|
||||
|
||||
event_location_data['external_id'] = event_location_external_id # NOTE: This should never change
|
||||
|
||||
log.debug(event_location_data)
|
||||
|
||||
if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'):
|
||||
log.debug(event_location_obj_in_result)
|
||||
event_location_id = event_location_obj_in_result
|
||||
else:
|
||||
log.warning(event_location_obj_in_result)
|
||||
continue # Something unexpected may have happened
|
||||
data['event_location_id'] = event_location_id
|
||||
|
||||
|
||||
event_session_summary = {}
|
||||
event_session_summary['event_session_external_id'] = event_session.get('id')
|
||||
@@ -89,27 +161,26 @@ async def import_event_session_list(
|
||||
session_start_datetime_str = session_time_parts[0]+'T'+session_time_parts[1]
|
||||
session_end_datetime_str = session_time_parts[0]+'T'+session_time_parts[2]
|
||||
event_session_summary['event_session_start_datetime'] = session_start_datetime_str
|
||||
event_session_summary['event_session_end_datetime'] = session_end_datetime_str
|
||||
# event_session_summary['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
|
||||
# end_time???
|
||||
# event_session_summary['event_session_end_datetime'] = session_end_datetime_str
|
||||
event_session_summary['event_session_type'] = event_session.get('ProgramCode')
|
||||
event_session_summary['event_session_type_name'] = event_session.get('ProgramTitle')
|
||||
event_session_summary['event_session_subject_name'] = event_session.get('SubjectTitle')
|
||||
event_session_summary['event_session_file_list'] = event_session.get('Child_MediaFiles')
|
||||
event_session_summary['event_session_slot_data'] = event_session.get('ParentList_SlotData')
|
||||
event_session_summary['event_location_id'] = event_location_id
|
||||
|
||||
event_session_data = {}
|
||||
event_session_data['external_id'] = event_session.get('id')
|
||||
event_session_data['name'] = event_session.get('Title')
|
||||
event_session_data['start_datetime'] = session_start_datetime_str
|
||||
event_session_data['end_datetime'] = session_end_datetime_str
|
||||
# event_session_data['event_session_start_end_datetime'] = event_session.get('SortableTimeString')
|
||||
# end_time???
|
||||
event_session_data['event_location_id'] = event_location_id
|
||||
|
||||
# event_session_data['event_session_type'] = event_session.get('ProgramCode')
|
||||
# event_session_data['event_session_type_name'] = event_session.get('ProgramTitle')
|
||||
# event_session_data['event_session_subject_name'] = event_session.get('SubjectTitle')
|
||||
# event_session_data['event_session_file_list'] = event_session.get('Child_MediaFiles')
|
||||
|
||||
|
||||
# SortableTimeString - '2023-05-11_08:00:00_09:30:00'
|
||||
# ParentList_Day - ['Day/2023-05-11']
|
||||
# Child_MediaFiles
|
||||
@@ -187,7 +258,8 @@ async def import_event_session_list(
|
||||
if confex_session_detail:
|
||||
# log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
# log.setLevel(logging.DEBUG)
|
||||
log.info(f'Session Detail: {confex_session_detail}')
|
||||
log.info(f'Got session detail for Confex session {confex_session_id}')
|
||||
log.debug(f'Session Detail: {confex_session_detail}')
|
||||
|
||||
# Presentation List
|
||||
event_session_summary['event_presentation_list'] = []
|
||||
@@ -197,13 +269,13 @@ async def import_event_session_list(
|
||||
# Presentation Information
|
||||
confex_presentation_id = confex_presentation_paper
|
||||
|
||||
if confex_presentation_id in ('Paper/1928'): # '', ''
|
||||
log.setLevel(logging.DEBUG)
|
||||
log.warning(f'**************** FOUND {confex_presentation_id} ******************')
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
log.warning(f'*** FOUND {confex_presentation_id} ***')
|
||||
continue
|
||||
# if confex_presentation_id in ('Paper/1928', 'Paper/1770', 'Paper/1214'): # '', ''
|
||||
# log.setLevel(logging.DEBUG)
|
||||
# log.warning(f'**************** FOUND {confex_presentation_id} ******************')
|
||||
# else:
|
||||
# log.setLevel(logging.INFO)
|
||||
# log.warning(f'*** FOUND {confex_presentation_id} ***')
|
||||
# continue
|
||||
|
||||
event_presentation_data = {}
|
||||
event_presentation_data['external_id'] = confex_presentation_id
|
||||
@@ -287,34 +359,8 @@ async def import_event_session_list(
|
||||
|
||||
|
||||
|
||||
# Presentation File List (not Presenter specific???)
|
||||
log.info('Get presentation files for a presenter...')
|
||||
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
|
||||
event_presentation_data['event_file_list'] = []
|
||||
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(confex_vendor_file)
|
||||
confex_file_id = confex_vendor_file
|
||||
|
||||
|
||||
log.info('Confex: Getting file detail...')
|
||||
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
|
||||
event_file_data = {}
|
||||
if confex_file_detail:
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(f'File Detail: {confex_file_detail}')
|
||||
|
||||
event_file_data['filename'] = confex_file_detail.get('FileName')
|
||||
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
|
||||
event_file_data['ext'] = confex_file_detail.get('Ext')
|
||||
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
|
||||
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
|
||||
event_file_data['url'] = confex_file_detail.get('URL')
|
||||
event_file_data['created_on'] = confex_file_detail.get('_originated')
|
||||
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
|
||||
|
||||
event_presentation_data['event_file_list'].append(event_file_data)
|
||||
|
||||
|
||||
# Presenter Information
|
||||
submitter_full_name = confex_presentation_detail.get('SubmitterFullName')
|
||||
@@ -393,10 +439,10 @@ async def import_event_session_list(
|
||||
|
||||
try:
|
||||
event_presenter_obj = Event_Presenter_Base(**event_presenter_data_sql)
|
||||
log.warning(event_presenter_obj)
|
||||
except ValidationError as e:
|
||||
log.error(e.json())
|
||||
return False
|
||||
log.debug(event_presenter_obj)
|
||||
|
||||
event_presenter_id = None
|
||||
if event_presenter_result := sql_select(sql=sql_select_event_presenter, data=event_presenter_qry):
|
||||
@@ -439,6 +485,44 @@ async def import_event_session_list(
|
||||
break
|
||||
# return False
|
||||
|
||||
# Presentation File List for Presenter
|
||||
log.info('Get presentation files for a presenter...')
|
||||
log.debug(confex_presentation_detail.get('ChildList_VendorFiles'))
|
||||
event_presentation_data['confex_file_list'] = []
|
||||
event_presentation_data['event_file_list'] = []
|
||||
for confex_vendor_file in confex_presentation_detail.get('ChildList_VendorFiles'):
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(confex_vendor_file)
|
||||
confex_file_id = confex_vendor_file
|
||||
|
||||
log.info('Confex: Getting file detail...')
|
||||
confex_file_detail = get_event_file_detail(confex_file_id = confex_file_id)
|
||||
event_file_data = {}
|
||||
file_info = {} # For locally saved file info below
|
||||
if confex_file_detail:
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(f'File Detail: {confex_file_detail}')
|
||||
|
||||
event_file_data['filename'] = confex_file_detail.get('FileName')
|
||||
event_file_data['display_name'] = confex_file_detail.get('DisplayName')
|
||||
event_file_data['extension'] = confex_file_detail.get('Ext')
|
||||
event_file_data['size'] = confex_file_detail.get('size') # in bytes?
|
||||
event_file_data['file_purpose'] = confex_file_detail.get('table') # 'Handout', 'Presentation'
|
||||
event_file_data['url'] = confex_file_detail.get('URL')
|
||||
event_file_data['created_on'] = confex_file_detail.get('_originated')
|
||||
event_file_data['updated_on'] = confex_file_detail.get('_lastchanged')
|
||||
|
||||
file_info = await get_event_file_save_local(
|
||||
url = confex_file_detail.get('URL'),
|
||||
confex_file_info = event_file_data,
|
||||
account_id = account_id,
|
||||
link_to_type = 'event_presenter',
|
||||
link_to_id = event_presenter_id,
|
||||
)
|
||||
|
||||
event_presentation_data['confex_file_list'].append(event_file_data)
|
||||
event_presentation_data['event_file_list'].append(file_info)
|
||||
|
||||
# event_presenter_data['event_presenter_id'] = event_presenter_id # _random
|
||||
if len(event_presentation_data['event_presenter_list']):
|
||||
event_presentation_data['event_presenter_list'][0]['event_presenter_id'] = event_presenter_id # _random
|
||||
|
||||
@@ -331,7 +331,7 @@ async def event_importing_create_update_w_external_id(
|
||||
# ### BEGIN ### Event Importing ### event_importing_program_data() ###
|
||||
# Based on the program import template the clients are given.
|
||||
# Create and update locations, sessions, presentations, and presenters as needed.
|
||||
# Updated 2022-10-07
|
||||
# Updated 2023-05-01
|
||||
@router.post('/event/{event_id}/importing/program_data', response_model=Resp_Body_Base)
|
||||
async def event_importing_program_data(
|
||||
event_id: str = Query(..., min_length=11, max_length=22),
|
||||
@@ -410,6 +410,7 @@ async def event_importing_program_data(
|
||||
|
||||
'location_external_id': str, 'location_code': str, 'location_sort': int,
|
||||
'presentation_external_id': str, 'presentation_code': str, 'presentation_sort': int,
|
||||
|
||||
'presenter_external_id': str, 'presenter_code': str, 'presenter_number': int, 'presenter_designations': str, 'presenter_sort': int,
|
||||
'session_external_id': str, 'session_code': str, 'session_sort': int,
|
||||
|
||||
@@ -441,19 +442,22 @@ async def event_importing_program_data(
|
||||
'Affiliations': 'presenter_presenter_affiliations',
|
||||
'Company': 'presenter_presenter_affiliations',
|
||||
|
||||
'email': 'presenter_email',
|
||||
'Email': 'presenter_email',
|
||||
'Email Address': 'presenter_email',
|
||||
'CC Email Address': 'presenter_cc_email',
|
||||
|
||||
'location_title': 'location_name',
|
||||
'Location Code': 'location_code',
|
||||
'session_location': 'location_code',
|
||||
'Session Location': 'location_code',
|
||||
'location_title': 'location_name',
|
||||
'Location Name': 'location_name',
|
||||
'session_location': 'location_name',
|
||||
'Session Location': 'location_name',
|
||||
# 'session_location': 'location_name',
|
||||
|
||||
'session_title': 'session_name',
|
||||
'Session Code': 'session_code',
|
||||
'Session Name': 'session_name',
|
||||
'Session Hide': 'hide_session',
|
||||
|
||||
'presentation_title': 'presentation_name',
|
||||
'Presentation Code': 'presentation_code',
|
||||
@@ -462,6 +466,9 @@ async def event_importing_program_data(
|
||||
'Presenter Code': 'presenter_code',
|
||||
'Presenter Number': 'presenter_number', # for sorting
|
||||
'Presenter Name': 'presenter_name',
|
||||
|
||||
'Presenter Notes': 'presenter_notes',
|
||||
'Notes': 'presenter_notes',
|
||||
},
|
||||
inplace = True)
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
@@ -488,6 +495,7 @@ async def event_importing_program_data(
|
||||
else:
|
||||
log.warning('No location external ID or code was found. Creating one with the name...')
|
||||
event_location_name = record.get('location_name', '').strip()
|
||||
# event_location_code = event_location_name
|
||||
event_location_external_id = f'{event_location_name}' # .strip()
|
||||
log.debug(f'Event Location External ID: {event_location_external_id}')
|
||||
|
||||
@@ -547,7 +555,10 @@ async def event_importing_program_data(
|
||||
event_location_data = {}
|
||||
event_location_data['code'] = record.get('location_code')
|
||||
event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case
|
||||
event_location_data['name'] = record.get('location_name')
|
||||
if record.get('location_name'):
|
||||
event_location_data['name'] = record.get('location_name')
|
||||
elif record.get('location_code'):
|
||||
event_location_data['name'] = record.get('location_code')
|
||||
event_location_data['enable'] = True
|
||||
|
||||
event_location_id = None
|
||||
@@ -613,6 +624,9 @@ async def event_importing_program_data(
|
||||
|
||||
event_session_data['sort'] = record.get('session_sort')
|
||||
|
||||
if record.get('hide_session') in (True, 1, 'yes', 'Yes', 'YES'):
|
||||
event_session_data['hide'] = True
|
||||
|
||||
event_session_data['enable'] = True
|
||||
event_session_data['notes'] = record.get('session_notes')
|
||||
|
||||
@@ -670,15 +684,19 @@ async def event_importing_program_data(
|
||||
event_presentation_data['name'] = record.get('presentation_name')
|
||||
event_presentation_data['description'] = record.get('presentation_description')
|
||||
|
||||
event_presentation_data['start_datetime'] = record.get('presentation_start_datetime')
|
||||
# event_presentation_start_datetime = record.get('presentation_start_date', '') + ' ' + record.get('presentation_start_time', '')
|
||||
# event_presentation_data['start_datetime'] = event_presentation_start_datetime
|
||||
data['presentation_start_datetime'] = event_presentation_data['start_datetime']
|
||||
if record.get('presentation_start_datetime'):
|
||||
event_presentation_data['start_datetime'] = record.get('presentation_start_datetime')
|
||||
data['presentation_start_datetime'] = event_presentation_data['start_datetime']
|
||||
else:
|
||||
event_presentation_data['start_datetime'] = None
|
||||
data['presentation_start_datetime'] = None
|
||||
|
||||
event_presentation_data['end_datetime'] = record.get('presentation_end_datetime')
|
||||
# event_presentation_end_datetime = record.get('presentation_end_date', '') + ' ' + record.get('presentation_end_time', '')
|
||||
# event_presentation_data['end_datetime'] = event_presentation_end_datetime
|
||||
data['presentation_end_datetime'] = event_presentation_data['end_datetime']
|
||||
if record.get('presentation_end_datetime'):
|
||||
event_presentation_data['end_datetime'] = record.get('presentation_end_datetime')
|
||||
data['presentation_end_datetime'] = event_presentation_data['end_datetime']
|
||||
else:
|
||||
event_presentation_data['end_datetime'] = None
|
||||
data['presentation_end_datetime'] = None
|
||||
|
||||
event_presentation_data['sort'] = record.get('presentation_sort')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user