Prep for AAPOR with Confex and NCSD STD. General clean up.
This commit is contained in:
Scott Idem
@@ -44,7 +44,7 @@ def load_hosted_file_obj(
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_hosted_file_link_list: bool = False,
|
||||
) -> Hosted_File_Base|dict|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
@@ -116,7 +116,7 @@ def lookup_file_hash(
|
||||
# ### END ### API Hosted File Methods ### lookup_file_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### get_file_object_hash() ###
|
||||
@logger_reset
|
||||
async def get_file_object_hash(file_object:File):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
@@ -137,25 +137,25 @@ async def get_file_object_hash(file_object:File):
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
|
||||
return file_hash
|
||||
# ### END ### API Hosted File Route ### get_file_object_hash() ###
|
||||
# ### END ### API Hosted File Methods ### get_file_object_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### guess_file_extension() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### guess_file_extension() ###
|
||||
def guess_file_extension(filename: str):
|
||||
return filename.rsplit('.', 1)[1].lower()
|
||||
# ### END ### API Hosted File Route ### guess_file_extension() ###
|
||||
# ### END ### API Hosted File Methods ### guess_file_extension() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### allowed_file_extension() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### allowed_file_extension() ###
|
||||
def allowed_file_extension(extension: str, extension_list: list):
|
||||
return extension.lower() in extension_list # app.config['ALLOWED_EXTENSIONS']
|
||||
# ### END ### API Hosted File Route ### allowed_file_extension() ###
|
||||
# ### END ### API Hosted File Methods ### allowed_file_extension() ###
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### save_file() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### save_file() ###
|
||||
# Updated 2022-08-09
|
||||
@logger_reset
|
||||
async def save_file(
|
||||
@@ -167,7 +167,7 @@ async def save_file(
|
||||
link_to_id_random: str = None,
|
||||
check_allowed_extension: bool = False,
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
@@ -371,7 +371,134 @@ async def save_file(
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Route ### save_file() ###
|
||||
# ### END ### API Hosted File Methods ### save_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### save_file_to_hosted_file() ###
|
||||
# Updated 2022-08-09
|
||||
@logger_reset
|
||||
async def save_file_to_hosted_file(
|
||||
file_path: str,
|
||||
filename: str,
|
||||
extension: str,
|
||||
account_id: int,
|
||||
link_to_type: str,
|
||||
link_to_id: int,
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
log.debug(file_path)
|
||||
log.debug(f'Filename: {filename} Extension: {extension}')
|
||||
|
||||
file_obj = open(file_path, 'rb')
|
||||
|
||||
|
||||
file_info: dict = {}
|
||||
file_info['saved'] = None
|
||||
file_info['link_to_type'] = link_to_type
|
||||
file_info['link_to_id'] = link_to_id
|
||||
file_info['filename'] = filename
|
||||
file_info['extension'] = extension # guess_file_extension(filename=filename)
|
||||
|
||||
# if check_allowed_extension:
|
||||
# if allowed_file_extension(extension=file_info['extension'], extension_list=['jpg','png','webp']):
|
||||
# file_info['extension_allowed'] = True
|
||||
# else:
|
||||
# file_info['extension_allowed'] = False
|
||||
# file_info['saved'] = False
|
||||
# return file_info
|
||||
# else:
|
||||
# file_info['extension_allowed'] = None
|
||||
|
||||
# There is a difference between Content-Type and MIME type.
|
||||
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
|
||||
# file_info['content_type'] = file.content_type # might also include charset or other parameters
|
||||
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
|
||||
|
||||
file_obj.seek(0, os.SEEK_END)
|
||||
file_size = file_obj.tell()
|
||||
file_obj.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
log.debug(file_size)
|
||||
file_info['size'] = file_size
|
||||
|
||||
file_hash = await get_file_object_hash(file_obj)
|
||||
log.debug(file_hash)
|
||||
file_info['hash_sha256'] = file_hash
|
||||
|
||||
# 16384 bytes is the default
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
buffer_size = 524288
|
||||
|
||||
#f_src = open(file_src, 'rb')
|
||||
f_src = file_obj # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||
|
||||
file_hash_subdirectory = file_hash[0:2]
|
||||
subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory)
|
||||
log.debug(subdirectory_dest)
|
||||
pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True)
|
||||
file_info['subdirectory_path'] = file_hash_subdirectory
|
||||
|
||||
#file_dest = f'{hosted_files_path}{file.filename}'
|
||||
# file_dest = f'{hosted_files_path}{file_hash}.file'
|
||||
|
||||
file_dest = os.path.join(hosted_files_path, f'{file_hash}.file')
|
||||
file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file')
|
||||
|
||||
existing_file_check = pathlib.Path(file_dest)
|
||||
existing_file_check_subdir = pathlib.Path(file_dest_w_subdir)
|
||||
|
||||
log.debug(existing_file_check_subdir)
|
||||
# return file_info
|
||||
|
||||
|
||||
if existing_file_check_subdir.exists():
|
||||
log.warning('This file already exists at the destination with the subdirectory. Not re-saving.')
|
||||
file_info['already_exists'] = True
|
||||
file_info['already_exists_subdir'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.warning('This file does not already exist at the destination subdirectory.')
|
||||
file_info['already_exists'] = False
|
||||
file_info['already_exists_subdir'] = False
|
||||
try:
|
||||
log.info('Saving file to destination...')
|
||||
f_dest = open(file_dest_w_subdir, 'wb')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
|
||||
log.info(f'File saved to: {hosted_files_path}')
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
log.exception('***')
|
||||
log.exception(str(e))
|
||||
log.exception('^^^ exception ^^^')
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
return False
|
||||
log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}')
|
||||
log.info(f"Filename: {file_info['filename']}")
|
||||
log.info(f"Subdirectory Path: {file_info['subdirectory_path']}")
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(file_info)
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Methods ### save_file_to_hosted_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||
|
||||
Reference in New Issue
Block a user