Work on file uploads, hosted files, deletion of links, records, and stored files.
This commit is contained in:
Scott Idem
@@ -1,12 +1,11 @@
|
||||
from __future__ import annotations
|
||||
import datetime, hashlib, os, pathlib, shutil, time
|
||||
|
||||
from fastapi import File, UploadFile
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
||||
from app.lib_general import log, logging
|
||||
from app.db_sql import redis_lookup_id_random, sql_enable_part, sql_insert, sql_limit_offset_part, sql_select, sql_update
|
||||
from app.lib_general import log, logging, logger_reset
|
||||
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
|
||||
@@ -129,9 +128,11 @@ async def save_file(
|
||||
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
|
||||
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
log.debug(dir(file))
|
||||
log.debug(f'{file.filename}')
|
||||
@@ -176,26 +177,37 @@ async def save_file(
|
||||
#f_src = open(file_src, 'rb')
|
||||
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||
|
||||
#file_dest = f'{hosted_file_path}{file.filename}'
|
||||
file_dest = f'{hosted_file_path}{file_hash}.file'
|
||||
file_hash_subdirectory = file_hash[0:2]
|
||||
subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory)
|
||||
log.debug(subdirectory_dest)
|
||||
pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True)
|
||||
file_info['subdirectory_path'] = file_hash_subdirectory
|
||||
|
||||
#file_dest = f'{hosted_files_path}{file.filename}'
|
||||
# file_dest = f'{hosted_files_path}{file_hash}.file'
|
||||
|
||||
file_dest = os.path.join(hosted_files_path, f'{file_hash}.file')
|
||||
file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file')
|
||||
|
||||
existing_file_check = pathlib.Path(file_dest)
|
||||
existing_file_check_subdir = pathlib.Path(file_dest_w_subdir)
|
||||
|
||||
|
||||
if existing_file_check.exists():
|
||||
log.warning('This file already exists at the destination without the subdirectory. Not re-saving. Going to move the current file and update the database later.')
|
||||
file_info['already_exists'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
file_info['already_exists'] = False
|
||||
file_info['already_exists_subdir'] = False
|
||||
try:
|
||||
f_dest = open(file_dest, 'wb')
|
||||
log.info('Moving file to sub directory destination...')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
shutil.move(existing_file_check, existing_file_check_subdir)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
|
||||
log.info(f'File moved to: {hosted_files_path}')
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
@@ -205,13 +217,79 @@ async def save_file(
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
elif existing_file_check_subdir.exists():
|
||||
log.warning('This file already exists at the destination with the subdirectory. Not re-saving.')
|
||||
file_info['already_exists'] = True
|
||||
file_info['already_exists_subdir'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.warning('This file does not already exist at the destination with or without the subdirectory.')
|
||||
file_info['already_exists'] = False
|
||||
file_info['already_exists_subdir'] = False
|
||||
try:
|
||||
log.info('Saving file to destination...')
|
||||
f_dest = open(file_dest_w_subdir, 'wb')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
|
||||
log.info(f'File saved to: {hosted_files_path}')
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
log.exception('***')
|
||||
log.exception(str(e))
|
||||
log.exception('^^^ exception ^^^')
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
return False
|
||||
log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}')
|
||||
log.info(f"Filename: {file_info['filename']}")
|
||||
log.info(f"Subdirectory Path: {file_info['subdirectory_path']}")
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(file_info)
|
||||
|
||||
|
||||
# if existing_file_check.exists():
|
||||
# file_info['already_exists'] = True
|
||||
# file_info['copy_timer'] = 0
|
||||
# file_info['saved'] = True
|
||||
# else:
|
||||
# file_info['already_exists'] = False
|
||||
# try:
|
||||
# f_dest = open(file_dest, 'wb')
|
||||
# timer_start = time.process_time()
|
||||
# shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
# timer_end = time.process_time()
|
||||
# elapsed_time = timer_end - timer_start
|
||||
# log.debug(f'Elapsed time: {elapsed_time}')
|
||||
# file_info['copy_timer'] = elapsed_time
|
||||
# file_info['saved'] = True
|
||||
# except Exception as e:
|
||||
# log.exception('*** An exception happened. ***')
|
||||
# log.exception(repr(e))
|
||||
# log.exception('***')
|
||||
# log.exception(str(e))
|
||||
# log.exception('^^^ exception ^^^')
|
||||
|
||||
# file_info['copy_timer'] = 0
|
||||
# file_info['saved'] = False
|
||||
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_files_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Route ### save_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### hosted_file_link() ###
|
||||
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||
def create_hosted_file_link(
|
||||
account_id: int|str,
|
||||
hosted_file_id: int|str,
|
||||
@@ -252,8 +330,98 @@ def create_hosted_file_link(
|
||||
|
||||
log.debug(hosted_file_link_data_in_result)
|
||||
return True
|
||||
# ### END ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||
|
||||
# ### END ### API Hosted File Route ### hosted_file_link() ###
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### delete_hosted_file() ###
|
||||
# Updated 2022-08-08
|
||||
@logger_reset
|
||||
def delete_hosted_file(
|
||||
account_id: int|str,
|
||||
hosted_file_id: int|str,
|
||||
|
||||
link_to_type: str = None,
|
||||
link_to_id: int|str = None,
|
||||
|
||||
rm_all_links: bool = False,
|
||||
rm_orphan: bool = False,
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||
# else: return False
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return False
|
||||
|
||||
sql = f"""
|
||||
DELETE FROM hosted_file
|
||||
WHERE hosted_file_id = :hosted_file_id
|
||||
"""
|
||||
hosted_file_data = {}
|
||||
hosted_file_data['hosted_file_id'] = hosted_file_id
|
||||
# hosted_file_data['link_to_type'] = link_to_type
|
||||
# hosted_file_data['link_to_id'] = link_to_id
|
||||
log.debug(hosted_file_data)
|
||||
|
||||
if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_data):
|
||||
log.info(f'Deleted Hosted File Link. Hosted File ID: {hosted_file_id}')
|
||||
elif hosted_file_delete_result is None:
|
||||
return None
|
||||
else:
|
||||
return False
|
||||
|
||||
return True
|
||||
# ### END ### API Hosted File Methods ### delete_hosted_file() ###
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### delete_hosted_file_link() ###
|
||||
# Updated 2022-08-08
|
||||
@logger_reset
|
||||
def delete_hosted_file_link(
|
||||
account_id: int|str,
|
||||
hosted_file_id: int|str,
|
||||
|
||||
link_to_type: str,
|
||||
link_to_id: int|str,
|
||||
|
||||
rm_orphan: bool = False,
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||
# else: return False
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return False
|
||||
|
||||
if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass
|
||||
else: return False
|
||||
|
||||
sql = f"""
|
||||
DELETE FROM hosted_file_link
|
||||
WHERE hosted_file_id = :hosted_file_id
|
||||
AND link_to_type = :link_to_type
|
||||
AND link_to_id = :link_to_id
|
||||
"""
|
||||
hosted_file_link_data = {}
|
||||
hosted_file_link_data['hosted_file_id'] = hosted_file_id
|
||||
hosted_file_link_data['link_to_type'] = link_to_type
|
||||
hosted_file_link_data['link_to_id'] = link_to_id
|
||||
log.debug(hosted_file_link_data)
|
||||
|
||||
if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_link_data):
|
||||
log.info(f'Deleted Hosted File Link. Hosted File ID: {hosted_file_id}, Link To Type: {link_to_type}, Link To ID: {link_to_id}')
|
||||
elif hosted_file_delete_result is None:
|
||||
return None
|
||||
else:
|
||||
return False
|
||||
|
||||
return True
|
||||
# ### END ### API Hosted File Methods ### delete_hosted_file_link() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||
@@ -310,3 +478,42 @@ def get_hosted_file_rec_list(
|
||||
|
||||
return hosted_file_rec_li
|
||||
# ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### get_hosted_file_link_rec_list() ###
|
||||
def get_hosted_file_link_rec_list(
|
||||
hosted_file_id: int|str,
|
||||
|
||||
link_to_type: str = None,
|
||||
link_to_id: int|str = None,
|
||||
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
) -> list|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
data = {'hosted_file_id': hosted_file_id}
|
||||
|
||||
# sql_enabled, data['enable'] = sql_enable_part(table_name='hosted_file', enabled=enabled) # Reasonably safe return str and bool
|
||||
sql_limit = sql_limit_offset_part(limit=limit, offset=offset) # Reasonably safe return str
|
||||
|
||||
sql = f"""
|
||||
SELECT *
|
||||
FROM `v_hosted_file` AS `hosted_file`
|
||||
WHERE
|
||||
`hosted_file`.hosted_file_id = :hosted_file_id
|
||||
ORDER BY `hosted_file`.created_on DESC, `hosted_file`.updated_on DESC
|
||||
{sql_limit};
|
||||
"""
|
||||
|
||||
if hosted_file_rec_li_result := sql_select(data=data, sql=sql, as_list=True):
|
||||
hosted_file_rec_li = hosted_file_rec_li_result
|
||||
else:
|
||||
hosted_file_rec_li = []
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(hosted_file_rec_li_result)
|
||||
|
||||
return hosted_file_rec_li
|
||||
# ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||
|
||||
Reference in New Issue
Block a user