Work on file uploads, hosted files, deletion of links, records, and stored files.
This commit is contained in:
@@ -1,12 +1,11 @@
|
|||||||
from __future__ import annotations
|
|
||||||
import datetime, hashlib, os, pathlib, shutil, time
|
import datetime, hashlib, os, pathlib, shutil, time
|
||||||
|
|
||||||
from fastapi import File, UploadFile
|
from fastapi import File, UploadFile
|
||||||
from typing import Dict, List, Optional, Set, Union
|
from typing import Dict, List, Optional, Set, Union
|
||||||
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
||||||
|
|
||||||
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
from app.db_sql import redis_lookup_id_random, sql_enable_part, sql_insert, sql_limit_offset_part, sql_select, sql_update
|
||||||
from app.lib_general import log, logging
|
from app.lib_general import log, logging, logger_reset
|
||||||
|
|
||||||
from app.models.hosted_file_models import Hosted_File_Base
|
from app.models.hosted_file_models import Hosted_File_Base
|
||||||
|
|
||||||
@@ -129,9 +128,11 @@ async def save_file(
|
|||||||
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
log.debug(locals())
|
log.debug(locals())
|
||||||
|
|
||||||
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
|
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||||
|
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||||
|
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||||
|
|
||||||
log.debug(shutil.disk_usage(hosted_file_path))
|
log.debug(shutil.disk_usage(hosted_files_path))
|
||||||
|
|
||||||
log.debug(dir(file))
|
log.debug(dir(file))
|
||||||
log.debug(f'{file.filename}')
|
log.debug(f'{file.filename}')
|
||||||
@@ -176,26 +177,37 @@ async def save_file(
|
|||||||
#f_src = open(file_src, 'rb')
|
#f_src = open(file_src, 'rb')
|
||||||
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
|
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||||
|
|
||||||
#file_dest = f'{hosted_file_path}{file.filename}'
|
file_hash_subdirectory = file_hash[0:2]
|
||||||
file_dest = f'{hosted_file_path}{file_hash}.file'
|
subdirectory_dest = os.path.join(hosted_files_path, file_hash_subdirectory)
|
||||||
|
log.debug(subdirectory_dest)
|
||||||
|
pathlib.Path(subdirectory_dest).mkdir(parents=True, exist_ok=True)
|
||||||
|
file_info['subdirectory_path'] = file_hash_subdirectory
|
||||||
|
|
||||||
|
#file_dest = f'{hosted_files_path}{file.filename}'
|
||||||
|
# file_dest = f'{hosted_files_path}{file_hash}.file'
|
||||||
|
|
||||||
|
file_dest = os.path.join(hosted_files_path, f'{file_hash}.file')
|
||||||
|
file_dest_w_subdir = os.path.join(subdirectory_dest, f'{file_hash}.file')
|
||||||
|
|
||||||
existing_file_check = pathlib.Path(file_dest)
|
existing_file_check = pathlib.Path(file_dest)
|
||||||
|
existing_file_check_subdir = pathlib.Path(file_dest_w_subdir)
|
||||||
|
|
||||||
|
|
||||||
if existing_file_check.exists():
|
if existing_file_check.exists():
|
||||||
|
log.warning('This file already exists at the destination without the subdirectory. Not re-saving. Going to move the current file and update the database later.')
|
||||||
file_info['already_exists'] = True
|
file_info['already_exists'] = True
|
||||||
file_info['copy_timer'] = 0
|
file_info['already_exists_subdir'] = False
|
||||||
file_info['saved'] = True
|
|
||||||
else:
|
|
||||||
file_info['already_exists'] = False
|
|
||||||
try:
|
try:
|
||||||
f_dest = open(file_dest, 'wb')
|
log.info('Moving file to sub directory destination...')
|
||||||
timer_start = time.process_time()
|
timer_start = time.process_time()
|
||||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
shutil.move(existing_file_check, existing_file_check_subdir)
|
||||||
timer_end = time.process_time()
|
timer_end = time.process_time()
|
||||||
elapsed_time = timer_end - timer_start
|
elapsed_time = timer_end - timer_start
|
||||||
log.debug(f'Elapsed time: {elapsed_time}')
|
log.debug(f'Elapsed time: {elapsed_time}')
|
||||||
file_info['copy_timer'] = elapsed_time
|
file_info['copy_timer'] = elapsed_time
|
||||||
file_info['saved'] = True
|
file_info['saved'] = True
|
||||||
|
|
||||||
|
log.info(f'File moved to: {hosted_files_path}')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log.exception('*** An exception happened. ***')
|
log.exception('*** An exception happened. ***')
|
||||||
log.exception(repr(e))
|
log.exception(repr(e))
|
||||||
@@ -205,13 +217,79 @@ async def save_file(
|
|||||||
|
|
||||||
file_info['copy_timer'] = 0
|
file_info['copy_timer'] = 0
|
||||||
file_info['saved'] = False
|
file_info['saved'] = False
|
||||||
log.debug(shutil.disk_usage(hosted_file_path))
|
elif existing_file_check_subdir.exists():
|
||||||
|
log.warning('This file already exists at the destination with the subdirectory. Not re-saving.')
|
||||||
|
file_info['already_exists'] = True
|
||||||
|
file_info['already_exists_subdir'] = True
|
||||||
|
file_info['copy_timer'] = 0
|
||||||
|
file_info['saved'] = True
|
||||||
|
else:
|
||||||
|
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.warning('This file does not already exist at the destination with or without the subdirectory.')
|
||||||
|
file_info['already_exists'] = False
|
||||||
|
file_info['already_exists_subdir'] = False
|
||||||
|
try:
|
||||||
|
log.info('Saving file to destination...')
|
||||||
|
f_dest = open(file_dest_w_subdir, 'wb')
|
||||||
|
timer_start = time.process_time()
|
||||||
|
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||||
|
timer_end = time.process_time()
|
||||||
|
elapsed_time = timer_end - timer_start
|
||||||
|
log.debug(f'Elapsed time: {elapsed_time}')
|
||||||
|
file_info['copy_timer'] = elapsed_time
|
||||||
|
file_info['saved'] = True
|
||||||
|
|
||||||
|
log.info(f'File saved to: {hosted_files_path}')
|
||||||
|
except Exception as e:
|
||||||
|
log.exception('*** An exception happened. ***')
|
||||||
|
log.exception(repr(e))
|
||||||
|
log.exception('***')
|
||||||
|
log.exception(str(e))
|
||||||
|
log.exception('^^^ exception ^^^')
|
||||||
|
|
||||||
|
file_info['copy_timer'] = 0
|
||||||
|
file_info['saved'] = False
|
||||||
|
return False
|
||||||
|
log.info(f'Disk usage: {shutil.disk_usage(hosted_files_path)}')
|
||||||
|
log.info(f"Filename: {file_info['filename']}")
|
||||||
|
log.info(f"Subdirectory Path: {file_info['subdirectory_path']}")
|
||||||
|
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(file_info)
|
||||||
|
|
||||||
|
|
||||||
|
# if existing_file_check.exists():
|
||||||
|
# file_info['already_exists'] = True
|
||||||
|
# file_info['copy_timer'] = 0
|
||||||
|
# file_info['saved'] = True
|
||||||
|
# else:
|
||||||
|
# file_info['already_exists'] = False
|
||||||
|
# try:
|
||||||
|
# f_dest = open(file_dest, 'wb')
|
||||||
|
# timer_start = time.process_time()
|
||||||
|
# shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||||
|
# timer_end = time.process_time()
|
||||||
|
# elapsed_time = timer_end - timer_start
|
||||||
|
# log.debug(f'Elapsed time: {elapsed_time}')
|
||||||
|
# file_info['copy_timer'] = elapsed_time
|
||||||
|
# file_info['saved'] = True
|
||||||
|
# except Exception as e:
|
||||||
|
# log.exception('*** An exception happened. ***')
|
||||||
|
# log.exception(repr(e))
|
||||||
|
# log.exception('***')
|
||||||
|
# log.exception(str(e))
|
||||||
|
# log.exception('^^^ exception ^^^')
|
||||||
|
|
||||||
|
# file_info['copy_timer'] = 0
|
||||||
|
# file_info['saved'] = False
|
||||||
|
|
||||||
|
|
||||||
|
log.debug(shutil.disk_usage(hosted_files_path))
|
||||||
|
|
||||||
return file_info
|
return file_info
|
||||||
# ### END ### API Hosted File Route ### save_file() ###
|
# ### END ### API Hosted File Route ### save_file() ###
|
||||||
|
|
||||||
|
|
||||||
# ### BEGIN ### API Hosted File Route ### hosted_file_link() ###
|
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||||
def create_hosted_file_link(
|
def create_hosted_file_link(
|
||||||
account_id: int|str,
|
account_id: int|str,
|
||||||
hosted_file_id: int|str,
|
hosted_file_id: int|str,
|
||||||
@@ -252,8 +330,98 @@ def create_hosted_file_link(
|
|||||||
|
|
||||||
log.debug(hosted_file_link_data_in_result)
|
log.debug(hosted_file_link_data_in_result)
|
||||||
return True
|
return True
|
||||||
|
# ### END ### API Hosted File Methods ### create_hosted_file_link() ###
|
||||||
|
|
||||||
# ### END ### API Hosted File Route ### hosted_file_link() ###
|
|
||||||
|
# ### BEGIN ### API Hosted File Methods ### delete_hosted_file() ###
|
||||||
|
# Updated 2022-08-08
|
||||||
|
@logger_reset
|
||||||
|
def delete_hosted_file(
|
||||||
|
account_id: int|str,
|
||||||
|
hosted_file_id: int|str,
|
||||||
|
|
||||||
|
link_to_type: str = None,
|
||||||
|
link_to_id: int|str = None,
|
||||||
|
|
||||||
|
rm_all_links: bool = False,
|
||||||
|
rm_orphan: bool = False,
|
||||||
|
):
|
||||||
|
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(locals())
|
||||||
|
|
||||||
|
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||||
|
# else: return False
|
||||||
|
|
||||||
|
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||||
|
else: return False
|
||||||
|
|
||||||
|
sql = f"""
|
||||||
|
DELETE FROM hosted_file
|
||||||
|
WHERE hosted_file_id = :hosted_file_id
|
||||||
|
"""
|
||||||
|
hosted_file_data = {}
|
||||||
|
hosted_file_data['hosted_file_id'] = hosted_file_id
|
||||||
|
# hosted_file_data['link_to_type'] = link_to_type
|
||||||
|
# hosted_file_data['link_to_id'] = link_to_id
|
||||||
|
log.debug(hosted_file_data)
|
||||||
|
|
||||||
|
if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_data):
|
||||||
|
log.info(f'Deleted Hosted File Link. Hosted File ID: {hosted_file_id}')
|
||||||
|
elif hosted_file_delete_result is None:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
# ### END ### API Hosted File Methods ### delete_hosted_file() ###
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# ### BEGIN ### API Hosted File Methods ### delete_hosted_file_link() ###
|
||||||
|
# Updated 2022-08-08
|
||||||
|
@logger_reset
|
||||||
|
def delete_hosted_file_link(
|
||||||
|
account_id: int|str,
|
||||||
|
hosted_file_id: int|str,
|
||||||
|
|
||||||
|
link_to_type: str,
|
||||||
|
link_to_id: int|str,
|
||||||
|
|
||||||
|
rm_orphan: bool = False,
|
||||||
|
):
|
||||||
|
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(locals())
|
||||||
|
|
||||||
|
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||||
|
# else: return False
|
||||||
|
|
||||||
|
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||||
|
else: return False
|
||||||
|
|
||||||
|
if link_to_id := redis_lookup_id_random(record_id_random=link_to_id, table_name=link_to_type): pass
|
||||||
|
else: return False
|
||||||
|
|
||||||
|
sql = f"""
|
||||||
|
DELETE FROM hosted_file_link
|
||||||
|
WHERE hosted_file_id = :hosted_file_id
|
||||||
|
AND link_to_type = :link_to_type
|
||||||
|
AND link_to_id = :link_to_id
|
||||||
|
"""
|
||||||
|
hosted_file_link_data = {}
|
||||||
|
hosted_file_link_data['hosted_file_id'] = hosted_file_id
|
||||||
|
hosted_file_link_data['link_to_type'] = link_to_type
|
||||||
|
hosted_file_link_data['link_to_id'] = link_to_id
|
||||||
|
log.debug(hosted_file_link_data)
|
||||||
|
|
||||||
|
if hosted_file_delete_result := sql_delete(sql=sql, data=hosted_file_link_data):
|
||||||
|
log.info(f'Deleted Hosted File Link. Hosted File ID: {hosted_file_id}, Link To Type: {link_to_type}, Link To ID: {link_to_id}')
|
||||||
|
elif hosted_file_delete_result is None:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return True
|
||||||
|
# ### END ### API Hosted File Methods ### delete_hosted_file_link() ###
|
||||||
|
|
||||||
|
|
||||||
# ### BEGIN ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
# ### BEGIN ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||||
@@ -310,3 +478,42 @@ def get_hosted_file_rec_list(
|
|||||||
|
|
||||||
return hosted_file_rec_li
|
return hosted_file_rec_li
|
||||||
# ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
# ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||||
|
|
||||||
|
|
||||||
|
# ### BEGIN ### API Hosted File Methods ### get_hosted_file_link_rec_list() ###
|
||||||
|
def get_hosted_file_link_rec_list(
|
||||||
|
hosted_file_id: int|str,
|
||||||
|
|
||||||
|
link_to_type: str = None,
|
||||||
|
link_to_id: int|str = None,
|
||||||
|
|
||||||
|
limit: int = 10,
|
||||||
|
offset: int = 0,
|
||||||
|
enabled: str = 'enabled', # enabled, disabled, all
|
||||||
|
) -> list|bool:
|
||||||
|
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(locals())
|
||||||
|
|
||||||
|
data = {'hosted_file_id': hosted_file_id}
|
||||||
|
|
||||||
|
# sql_enabled, data['enable'] = sql_enable_part(table_name='hosted_file', enabled=enabled) # Reasonably safe return str and bool
|
||||||
|
sql_limit = sql_limit_offset_part(limit=limit, offset=offset) # Reasonably safe return str
|
||||||
|
|
||||||
|
sql = f"""
|
||||||
|
SELECT *
|
||||||
|
FROM `v_hosted_file` AS `hosted_file`
|
||||||
|
WHERE
|
||||||
|
`hosted_file`.hosted_file_id = :hosted_file_id
|
||||||
|
ORDER BY `hosted_file`.created_on DESC, `hosted_file`.updated_on DESC
|
||||||
|
{sql_limit};
|
||||||
|
"""
|
||||||
|
|
||||||
|
if hosted_file_rec_li_result := sql_select(data=data, sql=sql, as_list=True):
|
||||||
|
hosted_file_rec_li = hosted_file_rec_li_result
|
||||||
|
else:
|
||||||
|
hosted_file_rec_li = []
|
||||||
|
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(hosted_file_rec_li_result)
|
||||||
|
|
||||||
|
return hosted_file_rec_li
|
||||||
|
# ### END ### API Hosted File Methods ### get_hosted_file_rec_list() ###
|
||||||
|
|||||||
@@ -176,7 +176,7 @@ async def download_event_file(
|
|||||||
else:
|
else:
|
||||||
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
|
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
|
||||||
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
|
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
|
||||||
# ### END ### API Hosted File ### download_tmp() ###
|
# ### END ### API Event File ### download_event_file() ###
|
||||||
|
|
||||||
|
|
||||||
# ### BEGIN ### API Event File ### get_event_file_obj() ###
|
# ### BEGIN ### API Event File ### get_event_file_obj() ###
|
||||||
|
|||||||
@@ -1,17 +1,16 @@
|
|||||||
# import datetime, hashlib, os, pathlib, shutil, time
|
import datetime, hashlib, os, pathlib, shutil, time
|
||||||
#from datetime import datetime, time, timedelta
|
|
||||||
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
|
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
|
||||||
from fastapi.responses import FileResponse
|
from fastapi.responses import FileResponse
|
||||||
from pydantic import BaseModel, EmailStr, Field
|
from pydantic import BaseModel, EmailStr, Field
|
||||||
from typing import Dict, List, Optional, Set, Union
|
from typing import Dict, List, Optional, Set, Union
|
||||||
|
|
||||||
from app.lib_general import log, logging
|
from app.lib_general import log, logging, common_route_params, Common_Route_Params, common_route_params_min, Common_Route_Params_Min
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
|
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
|
||||||
|
|
||||||
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||||
|
|
||||||
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link
|
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list
|
||||||
|
|
||||||
from app.models.hosted_file_models import Hosted_File_Base
|
from app.models.hosted_file_models import Hosted_File_Base
|
||||||
from app.models.response_models import Resp_Body_Base, mk_resp
|
from app.models.response_models import Resp_Body_Base, mk_resp
|
||||||
@@ -20,8 +19,62 @@ from app.models.response_models import Resp_Body_Base, mk_resp
|
|||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
# ### BEGIN ### API Hosted File ### download_hosted_file() ###
|
||||||
|
# Updated 2022-08-08
|
||||||
|
@router.get('/{hosted_file_id}/download', response_model=Resp_Body_Base)
|
||||||
|
async def download_hosted_file(
|
||||||
|
hosted_file_id: str = Query(..., min_length=11, max_length=22),
|
||||||
|
filename: str = Query(None, min_length=4, max_length=100),
|
||||||
|
|
||||||
|
commons: Common_Route_Params = Depends(common_route_params),
|
||||||
|
):
|
||||||
|
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(locals())
|
||||||
|
|
||||||
|
# ### SECTION ### Secondary data validation
|
||||||
|
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||||
|
else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
|
||||||
|
|
||||||
|
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||||
|
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||||
|
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||||
|
|
||||||
|
if hosted_file_obj := load_hosted_file_obj(
|
||||||
|
hosted_file_id = hosted_file_id,
|
||||||
|
# inc_hosted_file = True,
|
||||||
|
inc_hosted_file_link_list = True,
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
|
||||||
|
|
||||||
|
if not filename:
|
||||||
|
filename = hosted_file_obj.filename
|
||||||
|
log.info(f'Filename: {filename}')
|
||||||
|
dir_path = hosted_file_obj.directory_path
|
||||||
|
subdir_path = hosted_file_obj.subdirectory_path
|
||||||
|
hash_sha256 = hosted_file_obj.hash_sha256
|
||||||
|
hash_filename = hash_sha256+'.file'
|
||||||
|
|
||||||
|
if subdir_path:
|
||||||
|
full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
|
||||||
|
else:
|
||||||
|
full_subdirectory_path = hosted_files_path
|
||||||
|
log.debug(full_subdirectory_path)
|
||||||
|
pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
|
||||||
|
file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
|
||||||
|
log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
|
||||||
|
|
||||||
|
if os.path.exists(file_path_w_subdir):
|
||||||
|
return FileResponse(file_path_w_subdir, filename=filename)
|
||||||
|
else:
|
||||||
|
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
|
||||||
|
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
|
||||||
|
# ### END ### API Hosted File ### download_hosted_file() ###
|
||||||
|
|
||||||
|
|
||||||
# ### BEGIN ### API Hosted File Route ### upload_files() ###
|
# ### BEGIN ### API Hosted File Route ### upload_files() ###
|
||||||
# This just needs to return the currect model for a hosted_file
|
# This just needs to return the correct model for a hosted_file
|
||||||
# Everything else seems to be working well
|
# Everything else seems to be working well
|
||||||
# Should this also do something with meta data and updating the DB?
|
# Should this also do something with meta data and updating the DB?
|
||||||
@router.post('/upload_files')
|
@router.post('/upload_files')
|
||||||
@@ -66,9 +119,11 @@ async def upload_files(
|
|||||||
)
|
)
|
||||||
if file_info['saved']:
|
if file_info['saved']:
|
||||||
# Create a new host_file object entry
|
# Create a new host_file object entry
|
||||||
|
log.info('Check and create a new host_file object entry...')
|
||||||
if file_info['already_exists']:
|
if file_info['already_exists']:
|
||||||
# Look up in DB based on hash
|
# Look up in DB based on hash
|
||||||
# Get existing host_file object_entry and existing host_file.id_random.
|
# Get existing host_file object_entry and existing host_file.id_random.
|
||||||
|
log.info('Look up in DB based on hash...')
|
||||||
if hosted_file_sel_result := sql_select(
|
if hosted_file_sel_result := sql_select(
|
||||||
table_name = 'hosted_file',
|
table_name = 'hosted_file',
|
||||||
field_name = 'hash_sha256',
|
field_name = 'hash_sha256',
|
||||||
@@ -77,8 +132,41 @@ async def upload_files(
|
|||||||
hosted_file_id = hosted_file_sel_result.get('id_random', None)
|
hosted_file_id = hosted_file_sel_result.get('id_random', None)
|
||||||
# hosted_file_obj = Hosted_File_Base(**file_info)
|
# hosted_file_obj = Hosted_File_Base(**file_info)
|
||||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||||
|
|
||||||
|
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(hosted_file_dict)
|
||||||
|
|
||||||
|
# ******************************************************
|
||||||
|
# New as of 2021-08-26
|
||||||
|
|
||||||
|
# NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory.
|
||||||
|
|
||||||
|
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(file_info['already_exists'])
|
||||||
|
log.debug(file_info['already_exists_subdir'])
|
||||||
|
log.debug(file_info['subdirectory_path'])
|
||||||
|
|
||||||
|
if subdirectory_path := hosted_file_dict.get('subdirectory_path'):
|
||||||
|
log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}')
|
||||||
|
elif subdirectory_path := file_info.get('subdirectory_path', None):
|
||||||
|
log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}')
|
||||||
|
|
||||||
|
hosted_file_data_up = {}
|
||||||
|
hosted_file_data_up['id'] = hosted_file_id
|
||||||
|
hosted_file_data_up['subdirectory_path'] = subdirectory_path
|
||||||
|
|
||||||
|
if hosted_file_up_result := sql_update(
|
||||||
|
table_name = 'hosted_file',
|
||||||
|
data = hosted_file_data_up,
|
||||||
|
): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}')
|
||||||
else:
|
else:
|
||||||
# SOMETHING WENT WRONG
|
log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}')
|
||||||
|
log.debug(hosted_file_up_result)
|
||||||
|
else:
|
||||||
|
log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.')
|
||||||
|
# ******************************************************
|
||||||
|
else:
|
||||||
|
# NOTE: SOMETHING WENT WRONG
|
||||||
# Going to try and create a new host_file entry...
|
# Going to try and create a new host_file entry...
|
||||||
log.warning('For some reason a host_file object entry with the has was not found.')
|
log.warning('For some reason a host_file object entry with the has was not found.')
|
||||||
# file_info['id_random'] = None
|
# file_info['id_random'] = None
|
||||||
@@ -93,7 +181,8 @@ async def upload_files(
|
|||||||
log.debug(hosted_file_obj_result)
|
log.debug(hosted_file_obj_result)
|
||||||
log.debug(hosted_file_sel_result)
|
log.debug(hosted_file_sel_result)
|
||||||
else:
|
else:
|
||||||
# Just in case look up in DB based on hash
|
# NOTE: Just in case look up in DB based on hash
|
||||||
|
log.info('Look up in DB based on hash...')
|
||||||
if hosted_file_sel_result := sql_select(
|
if hosted_file_sel_result := sql_select(
|
||||||
table_name = 'hosted_file',
|
table_name = 'hosted_file',
|
||||||
field_name = 'hash_sha256',
|
field_name = 'hash_sha256',
|
||||||
@@ -129,9 +218,13 @@ async def upload_files(
|
|||||||
hosted_file_dict['already_exists'] = file_info['already_exists']
|
hosted_file_dict['already_exists'] = file_info['already_exists']
|
||||||
hosted_file_dict['saved'] = file_info['saved']
|
hosted_file_dict['saved'] = file_info['saved']
|
||||||
hosted_file_dict['copy_timer'] = file_info['copy_timer']
|
hosted_file_dict['copy_timer'] = file_info['copy_timer']
|
||||||
|
|
||||||
|
hosted_file_dict['filename'] = file_info['filename']
|
||||||
|
hosted_file_dict['extension'] = file_info['extension']
|
||||||
|
|
||||||
hosted_file_list.append(hosted_file_dict)
|
hosted_file_list.append(hosted_file_dict)
|
||||||
|
|
||||||
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
|
# NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
|
||||||
if create_hosted_file_link(
|
if create_hosted_file_link(
|
||||||
account_id = account_id,
|
account_id = account_id,
|
||||||
hosted_file_id = hosted_file_id,
|
hosted_file_id = hosted_file_id,
|
||||||
@@ -398,6 +491,127 @@ async def test_upload_files(
|
|||||||
return mk_resp(data=False, status_code=501, response=response)
|
return mk_resp(data=False, status_code=501, response=response)
|
||||||
|
|
||||||
|
|
||||||
|
# ### BEGIN ### API Hosted File ### delete_hosted_file() ###
|
||||||
|
# Updated 2022-08-08
|
||||||
|
@router.delete('/{hosted_file_id}', response_model=Resp_Body_Base)
|
||||||
|
async def delete_hosted_file(
|
||||||
|
hosted_file_id: str = Query(..., min_length=11, max_length=22),
|
||||||
|
|
||||||
|
link_to_type: str = None,
|
||||||
|
link_to_id: Union[int, str] = None,
|
||||||
|
|
||||||
|
rm_orphan: bool = False,
|
||||||
|
|
||||||
|
commons: Common_Route_Params = Depends(common_route_params),
|
||||||
|
):
|
||||||
|
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||||
|
log.debug(locals())
|
||||||
|
|
||||||
|
# ### SECTION ### Secondary data validation
|
||||||
|
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||||
|
else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
|
||||||
|
|
||||||
|
# ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE
|
||||||
|
# NOTE: If link_to_type and link_to_id passed then try and remove that link record first.
|
||||||
|
|
||||||
|
# if hosted_file_result := delete_hosted_file_link(
|
||||||
|
# account_id = commons.x_account_id,
|
||||||
|
# hosted_file_id = hosted_file_id,
|
||||||
|
|
||||||
|
# link_to_type = link_to_type,
|
||||||
|
# link_to_id = link_to_id,
|
||||||
|
|
||||||
|
# rm_orphan = rm_orphan,
|
||||||
|
# ):
|
||||||
|
# pass
|
||||||
|
# elif hosted_file_result is None:
|
||||||
|
# log.error('The hosted file link record was not found and may have already been deleted.')
|
||||||
|
# return mk_resp(data=False, status_code=404, response=commons.response) # Not Found
|
||||||
|
# else:
|
||||||
|
# log.error('Something went wrong while trying to delete the hosted file link record.')
|
||||||
|
# return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
|
||||||
|
|
||||||
|
# ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE
|
||||||
|
# NOTE: If not rm_orphan then do nothing else.
|
||||||
|
# NOTE: If rm_orphan then get list of links for file.
|
||||||
|
# NOTE: If 0 links result then delete the hosted_file record and file on the server.
|
||||||
|
# NOTE: If >0 links result then do nothing else.
|
||||||
|
|
||||||
|
# NOTE: Don't check or remove orphan
|
||||||
|
if not rm_orphan:
|
||||||
|
log.info('Remove hosted file link. No orphan check.')
|
||||||
|
return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
|
||||||
|
|
||||||
|
# NOTE: Check and remove orphan
|
||||||
|
if hosted_file_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id):
|
||||||
|
hosted_file_result_list = []
|
||||||
|
for hosted_file_rec in hosted_file_rec_list_result:
|
||||||
|
hosted_file_result_list.append(hosted_file_rec)
|
||||||
|
|
||||||
|
log.debug(hosted_file_result_list)
|
||||||
|
hosted_file_list = hosted_file_result_list
|
||||||
|
# NOT safe to delete the hosted_file record and file from server!!!
|
||||||
|
# STOP!
|
||||||
|
elif isinstance(hosted_file_rec_list_result, list):
|
||||||
|
hosted_file_list = []
|
||||||
|
# Safe to delete the hosted_file record and file from server???
|
||||||
|
# CONTINUE
|
||||||
|
else:
|
||||||
|
hosted_file_list = None
|
||||||
|
# Safe to delete the hosted_file record and file from server???
|
||||||
|
# CONTINUE
|
||||||
|
|
||||||
|
|
||||||
|
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||||
|
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||||
|
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||||
|
|
||||||
|
if hosted_file_obj := load_hosted_file_obj(
|
||||||
|
hosted_file_id = hosted_file_id,
|
||||||
|
# inc_hosted_file = True,
|
||||||
|
inc_hosted_file_link_list = rm_orphan, # if rm_orphan then need to include hosted_file_link_list
|
||||||
|
):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
|
||||||
|
|
||||||
|
# if not filename:
|
||||||
|
# filename = hosted_file_obj.filename
|
||||||
|
# log.info(f'Filename: {filename}')
|
||||||
|
dir_path = hosted_file_obj.directory_path
|
||||||
|
subdir_path = hosted_file_obj.subdirectory_path
|
||||||
|
hash_sha256 = hosted_file_obj.hash_sha256
|
||||||
|
hash_filename = hash_sha256+'.file'
|
||||||
|
|
||||||
|
if subdir_path:
|
||||||
|
full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
|
||||||
|
else:
|
||||||
|
full_subdirectory_path = hosted_files_path
|
||||||
|
log.debug(full_subdirectory_path)
|
||||||
|
# pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
|
||||||
|
file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
|
||||||
|
log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
|
||||||
|
|
||||||
|
if os.path.exists(file_path_w_subdir):
|
||||||
|
log.info('File exists!')
|
||||||
|
if rm_orphan:
|
||||||
|
log.info('Going remove the file if it is an orphan...')
|
||||||
|
# try:
|
||||||
|
# pathlib.Path(file_path_w_subdir).unlink()
|
||||||
|
# except OSError as e:
|
||||||
|
# log.error("Error: %s : %s" % (file_path, e.strerror))
|
||||||
|
# return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something went wrong while trying to delete the hosted file from the server.') # Bad Request
|
||||||
|
return mk_resp(data=True, response=commons.response, status_message='The orphan hosted file was deleted from the server.')
|
||||||
|
else:
|
||||||
|
log.info('Remove hosted file link. No orphan check.')
|
||||||
|
return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
|
||||||
|
|
||||||
|
else:
|
||||||
|
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
|
||||||
|
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
|
||||||
|
# ### END ### API Hosted File ### download_hosted_file() ###
|
||||||
|
|
||||||
|
|
||||||
# ### BEGIN ### API Hosted File ### get_hosted_file_obj() ###
|
# ### BEGIN ### API Hosted File ### get_hosted_file_obj() ###
|
||||||
# Updated 2021-09-07
|
# Updated 2021-09-07
|
||||||
@router.get('/{hosted_file_id}', response_model=Resp_Body_Base)
|
@router.get('/{hosted_file_id}', response_model=Resp_Body_Base)
|
||||||
|
|||||||
Reference in New Issue
Block a user