Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Work on file uploads, hosted files, deletion of links, records, and stored files.

Browse Source
This commit is contained in:
Scott Idem
2022-08-08 18:25:25 -04:00
parent 0dd80fd829
commit fac9ccad75
3 changed files with 446 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/routers/event_file.py
View File

@@ -176,7 +176,7 @@ async def download_event_file(
else:
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
# ### END ### API Hosted File ### download_tmp() ###
# ### END ### API Event File ### download_event_file() ###
# ### BEGIN ### API Event File ### get_event_file_obj() ###

230
app/routers/hosted_file.py
View File

@@ -1,17 +1,16 @@
# import datetime, hashlib, os, pathlib, shutil, time
#from datetime import datetime, time, timedelta
import datetime, hashlib, os, pathlib, shutil, time
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
from fastapi.responses import FileResponse
from pydantic import BaseModel, EmailStr, Field
from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging
from app.lib_general import log, logging, common_route_params, Common_Route_Params, common_route_params_min, Common_Route_Params_Min
from app.config import settings
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list
from app.models.hosted_file_models import Hosted_File_Base
from app.models.response_models import Resp_Body_Base, mk_resp
@@ -20,8 +19,62 @@ from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()
# ### BEGIN ### API Hosted File ### download_hosted_file() ###
# Updated 2022-08-08
@router.get('/{hosted_file_id}/download', response_model=Resp_Body_Base)
async def download_hosted_file(
hosted_file_id: str = Query(..., min_length=11, max_length=22),
filename: str = Query(None, min_length=4, max_length=100),
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# ### SECTION ### Secondary data validation
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
log.info(f'Hosted Files Path: {hosted_files_path}')
if hosted_file_obj := load_hosted_file_obj(
hosted_file_id = hosted_file_id,
# inc_hosted_file = True,
inc_hosted_file_link_list = True,
):
pass
else:
return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
if not filename:
filename = hosted_file_obj.filename
log.info(f'Filename: {filename}')
dir_path = hosted_file_obj.directory_path
subdir_path = hosted_file_obj.subdirectory_path
hash_sha256 = hosted_file_obj.hash_sha256
hash_filename = hash_sha256+'.file'
if subdir_path:
full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
else:
full_subdirectory_path = hosted_files_path
log.debug(full_subdirectory_path)
pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
if os.path.exists(file_path_w_subdir):
return FileResponse(file_path_w_subdir, filename=filename)
else:
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
# ### END ### API Hosted File ### download_hosted_file() ###
# ### BEGIN ### API Hosted File Route ### upload_files() ###
# This just needs to return the currect model for a hosted_file
# This just needs to return the correct model for a hosted_file
# Everything else seems to be working well
# Should this also do something with meta data and updating the DB?
@router.post('/upload_files')
@@ -66,9 +119,11 @@ async def upload_files(
)
if file_info['saved']:
# Create a new host_file object entry
log.info('Check and create a new host_file object entry...')
if file_info['already_exists']:
# Look up in DB based on hash
# Get existing host_file object_entry and existing host_file.id_random.
log.info('Look up in DB based on hash...')
if hosted_file_sel_result := sql_select(
table_name = 'hosted_file',
field_name = 'hash_sha256',
@@ -77,8 +132,41 @@ async def upload_files(
hosted_file_id = hosted_file_sel_result.get('id_random', None)
# hosted_file_obj = Hosted_File_Base(**file_info)
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(hosted_file_dict)
# ******************************************************
# New as of 2021-08-26
# NOTE: Working on moving all hosted files to subdirectories because there are a lot of files. The database needs to be updated if the file already exists and it does not exist in the new subdirectory.
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(file_info['already_exists'])
log.debug(file_info['already_exists_subdir'])
log.debug(file_info['subdirectory_path'])
if subdirectory_path := hosted_file_dict.get('subdirectory_path'):
log.info(f'The new subdirectory_path field was found in the database record? Subdirectory Path: {subdirectory_path}')
elif subdirectory_path := file_info.get('subdirectory_path', None):
log.info(f'The new subdirectory_path field was not found in the database record. This needs to be updated. Subdirectory Path: {subdirectory_path}')
hosted_file_data_up = {}
hosted_file_data_up['id'] = hosted_file_id
hosted_file_data_up['subdirectory_path'] = subdirectory_path
if hosted_file_up_result := sql_update(
table_name = 'hosted_file',
data = hosted_file_data_up,
): log.info(f'The hosted_file record has been updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}')
else:
log.warning(f'The hosted_file record was probably not updated with the new subdirectory_path. Hosted File ID: {hosted_file_id} Subdirectory Path: {subdirectory_path}')
log.debug(hosted_file_up_result)
else:
log.warning(f'The new subdirectory_path field was not found in the database record or the passed file info.')
# ******************************************************
else:
# SOMETHING WENT WRONG
# NOTE: SOMETHING WENT WRONG
# Going to try and create a new host_file entry...
log.warning('For some reason a host_file object entry with the has was not found.')
# file_info['id_random'] = None
@@ -93,7 +181,8 @@ async def upload_files(
log.debug(hosted_file_obj_result)
log.debug(hosted_file_sel_result)
else:
# Just in case look up in DB based on hash
# NOTE: Just in case look up in DB based on hash
log.info('Look up in DB based on hash...')
if hosted_file_sel_result := sql_select(
table_name = 'hosted_file',
field_name = 'hash_sha256',
@@ -129,9 +218,13 @@ async def upload_files(
hosted_file_dict['already_exists'] = file_info['already_exists']
hosted_file_dict['saved'] = file_info['saved']
hosted_file_dict['copy_timer'] = file_info['copy_timer']
hosted_file_dict['filename'] = file_info['filename']
hosted_file_dict['extension'] = file_info['extension']
hosted_file_list.append(hosted_file_dict)
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
# NOTE: Currently sql_insert does not handle all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
if create_hosted_file_link(
account_id = account_id,
hosted_file_id = hosted_file_id,
@@ -398,6 +491,127 @@ async def test_upload_files(
return mk_resp(data=False, status_code=501, response=response)
# ### BEGIN ### API Hosted File ### delete_hosted_file() ###
# Updated 2022-08-08
@router.delete('/{hosted_file_id}', response_model=Resp_Body_Base)
async def delete_hosted_file(
hosted_file_id: str = Query(..., min_length=11, max_length=22),
link_to_type: str = None,
link_to_id: Union[int, str] = None,
rm_orphan: bool = False,
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# ### SECTION ### Secondary data validation
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
# ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE
# NOTE: If link_to_type and link_to_id passed then try and remove that link record first.
# if hosted_file_result := delete_hosted_file_link(
# account_id = commons.x_account_id,
# hosted_file_id = hosted_file_id,
# link_to_type = link_to_type,
# link_to_id = link_to_id,
# rm_orphan = rm_orphan,
# ):
# pass
# elif hosted_file_result is None:
# log.error('The hosted file link record was not found and may have already been deleted.')
# return mk_resp(data=False, status_code=404, response=commons.response) # Not Found
# else:
# log.error('Something went wrong while trying to delete the hosted file link record.')
# return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
# ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE
# NOTE: If not rm_orphan then do nothing else.
# NOTE: If rm_orphan then get list of links for file.
# NOTE: If 0 links result then delete the hosted_file record and file on the server.
# NOTE: If >0 links result then do nothing else.
# NOTE: Don't check or remove orphan
if not rm_orphan:
log.info('Remove hosted file link. No orphan check.')
return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
# NOTE: Check and remove orphan
if hosted_file_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id):
hosted_file_result_list = []
for hosted_file_rec in hosted_file_rec_list_result:
hosted_file_result_list.append(hosted_file_rec)
log.debug(hosted_file_result_list)
hosted_file_list = hosted_file_result_list
# NOT safe to delete the hosted_file record and file from server!!!
# STOP!
elif isinstance(hosted_file_rec_list_result, list):
hosted_file_list = []
# Safe to delete the hosted_file record and file from server???
# CONTINUE
else:
hosted_file_list = None
# Safe to delete the hosted_file record and file from server???
# CONTINUE
# hosted_files_path = settings.FILES_PATH['hosted_files_root']
hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
log.info(f'Hosted Files Path: {hosted_files_path}')
if hosted_file_obj := load_hosted_file_obj(
hosted_file_id = hosted_file_id,
# inc_hosted_file = True,
inc_hosted_file_link_list = rm_orphan, # if rm_orphan then need to include hosted_file_link_list
):
pass
else:
return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
# if not filename:
# filename = hosted_file_obj.filename
# log.info(f'Filename: {filename}')
dir_path = hosted_file_obj.directory_path
subdir_path = hosted_file_obj.subdirectory_path
hash_sha256 = hosted_file_obj.hash_sha256
hash_filename = hash_sha256+'.file'
if subdir_path:
full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
else:
full_subdirectory_path = hosted_files_path
log.debug(full_subdirectory_path)
# pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
if os.path.exists(file_path_w_subdir):
log.info('File exists!')
if rm_orphan:
log.info('Going remove the file if it is an orphan...')
# try:
# pathlib.Path(file_path_w_subdir).unlink()
# except OSError as e:
# log.error("Error: %s : %s" % (file_path, e.strerror))
# return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something went wrong while trying to delete the hosted file from the server.') # Bad Request
return mk_resp(data=True, response=commons.response, status_message='The orphan hosted file was deleted from the server.')
else:
log.info('Remove hosted file link. No orphan check.')
return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
else:
log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
# ### END ### API Hosted File ### download_hosted_file() ###
# ### BEGIN ### API Hosted File ### get_hosted_file_obj() ###
# Updated 2021-09-07
@router.get('/{hosted_file_id}', response_model=Resp_Body_Base)