Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug fixes, clean up, stuff

Browse Source
This commit is contained in:
Scott Idem
2023-09-21 12:12:31 -04:00
parent fa39529f3f
commit 0ac9510546
4 changed files with 131 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
app/methods/hosted_file_methods.py
View File

@@ -144,7 +144,39 @@ def allowed_file_extension(extension: str, extension_list: list):
# ### END ### API Hosted File Methods ### allowed_file_extension() ###
# ### BEGIN ### API Hosted File Methods ### lookup_file_hash() ###
# Updated 2023-09-19
@logger_reset
def check_for_hosted_file_hash_file(
file_hash: str,
sub_dir: str,
) -> dict|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
file_size = None
hosted_files_path = settings.FILES_PATH['hosted_files_root']
log.info(f'Hosted Files Path: {hosted_files_path}')
log.debug(shutil.disk_usage(hosted_files_path))
hosted_files_dir_w_subdir = os.path.join(hosted_files_path, sub_dir)
path_hosted_files_dir_w_subdir = pathlib.Path(hosted_files_dir_w_subdir)
if path_hosted_files_dir_w_subdir.exists(): pass
else:
log.warning('Hashed hosted file subdirectory was not found in the hosted files root.')
return False
hosted_files_dir_w_subdir_filename = os.path.join(hosted_files_path, sub_dir, f'{file_hash}.file')
path_hosted_files_dir_w_subdir_filename = pathlib.Path(hosted_files_dir_w_subdir_filename)
if path_hosted_files_dir_w_subdir_filename.exists():
file_size = os.path.getsize(path_hosted_files_dir_w_subdir_filename)
else:
log.warning('Hashed hosted file not found in the expected hosted files subdirectory.')
return False
return {'found': True, 'file_size': file_size}
# ### BEGIN ### API Hosted File Methods ### save_file() ###

42
app/models/archive_content_models.py
View File

@@ -56,6 +56,8 @@ class Archive_Content_Base(BaseModel):
# xxxx_red: str = Field(default='xxx')
# xxxx_blue: str = Field(default_factory=testing)
hosted_file_path: str = None # '/testing/test-test'
api_hosted_file_path_download: str = None # '/testing/test-test'
api_hosted_file_path_stream: str = None # '/testing/test-test'
original_datetime: Optional[datetime.datetime]
original_datetime_timezone: Optional[str]
@@ -105,7 +107,7 @@ class Archive_Content_Base(BaseModel):
@validator('hosted_file_path', always=True)
def hosted_file_path_lookup(cls, v, values, **kwargs):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(v)
log.debug(values)
@@ -116,7 +118,43 @@ class Archive_Content_Base(BaseModel):
if filename := values.get('filename'):
path_str = f'{path_str}?filename={filename}'
log.debug(path_str)
log.info(f'Path: {path_str}')
return path_str
log.debug('NOT Found hosted_file_id_random...')
return v
@validator('api_hosted_file_path_download', always=True)
def api_hosted_file_path_download_lookup(cls, v, values, **kwargs):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(v)
log.debug(values)
if hosted_file_id_random := values.get('hosted_file_id_random'):
log.debug('Found hosted_file_id_random...')
path_str = f'/hosted_file/{hosted_file_id_random}/download'
if filename := values.get('filename'):
path_str = f'{path_str}?filename={filename}'
log.info(f'Path: {path_str}')
return path_str
log.debug('NOT Found hosted_file_id_random...')
return v
@validator('api_hosted_file_path_stream', always=True)
def api_hosted_file_path_stream_lookup(cls, v, values, **kwargs):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(v)
log.debug(values)
if hosted_file_id_random := values.get('hosted_file_id_random'):
log.debug('Found hosted_file_id_random...')
path_str = f'/hosted_file/{hosted_file_id_random}/stream'
if filename := values.get('filename'):
path_str = f'{path_str}?filename={filename}'
log.info(f'Path: {path_str}')
return path_str
log.debug('NOT Found hosted_file_id_random...')
return v

9
app/models/hosted_file_models.py
View File

@@ -61,6 +61,15 @@ class Hosted_File_Base(BaseModel):
created_on: Optional[datetime.datetime] = None
updated_on: Optional[datetime.datetime] = None
# Including convenience data
# This is only for convenience. Probably going to keep unless it causes a problem.
hosted_file_found_check: Optional[bool] = Field(
alias = 'found_check'
)
hosted_file_size_check: Optional[int] = Field( # File size in bytes
alias = 'size_check'
)
# Including other related objects
hosted_file_link_list: Optional[list] # Hosted_File_Base()

53
app/routers/hosted_file.py
View File

@@ -1,4 +1,4 @@
import aiofiles, datetime, hashlib, mimetypes, os, pathlib, shutil, time
import aiofiles, datetime, hashlib, mimetypes, os, pathlib, random, shutil, time
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
from fastapi.responses import FileResponse, StreamingResponse
# from fastapi.responses import StreamingResponse
@@ -14,7 +14,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, save_file_to_hosted_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, save_file_to_hosted_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash, check_for_hosted_file_hash_file
from app.models.hosted_file_models import Hosted_File_Base
from app.models.response_models import Resp_Body_Base, mk_resp
@@ -423,7 +423,7 @@ async def upload_files(
log.debug(locals())
# NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING
# time.sleep(3.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING
time.sleep(random.choice((3.5, 4.5, 5, 6.5))) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING
# NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING
account_id_random = account_id # This is for the account random str ID
@@ -894,6 +894,53 @@ async def get_hosted_file_obj(
# ### END ### API Hosted File ### get_hosted_file_obj() ###
# ### BEGIN ### API Hosted File ### get_hosted_file_obj_w_hash() ###
# Updated 2021-09-07
@router.get('/hash/{hosted_file_hash}', response_model=Resp_Body_Base)
async def check_hosted_file_obj_w_hash(
hosted_file_hash: str = Query(..., min_length=64, max_length=64), # Expects SHA256 hash
check_for_local: Optional[bool] = True,
commons: Common_Route_Params = Depends(common_route_params),
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if hosted_file_id := lookup_file_hash(
file_hash = hosted_file_hash,
):
hosted_file_dict = None
if hosted_file_obj := load_hosted_file_obj(
hosted_file_id = hosted_file_id,
):
hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset)
else:
# NOTE: This should not ever happen if the ID was already found.
return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash was found in the database, but something went wrong while loading the details from the database: Hosted File ID: {hosted_file_id}; Hosted File Hash: {hosted_file_hash}', response=commons.response) # Not Found
if check_for_local:
if check_for_hosted_file_hash_file_results := check_for_hosted_file_hash_file(
file_hash = hosted_file_hash,
sub_dir = hosted_file_obj.subdirectory_path,
):
hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset)
hosted_file_dict['hosted_file_found_check'] = True
hosted_file_dict['hosted_file_size_check'] = check_for_hosted_file_hash_file_results['file_size'] # File size in bytes
else:
return mk_resp(data=False, status_code=500, response=commons.response) # Bad Request
else:
return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash not found in the database: {hosted_file_hash}', response=commons.response) # Not Found
return mk_resp(data=hosted_file_dict, response=commons.response)
#return mk_resp(data=hosted_file_obj)
# ### END ### API Hosted File ### get_hosted_file_obj() ###
# ### BEGIN ### API Hosted File ### download_tmp() ###
# Updated 2023-04-05
@router.get('/tmp/{subdirectory}/{filename}/download', response_model=Resp_Body_Base)