From 0ac9510546e821695528fafab98235aa79387e3e Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 21 Sep 2023 12:12:31 -0400 Subject: [PATCH] Bug fixes, clean up, stuff --- app/methods/hosted_file_methods.py | 32 +++++++++++++++++ app/models/archive_content_models.py | 42 ++++++++++++++++++++-- app/models/hosted_file_models.py | 9 +++++ app/routers/hosted_file.py | 53 ++++++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 5 deletions(-) diff --git a/app/methods/hosted_file_methods.py b/app/methods/hosted_file_methods.py index 52212dd..5c777bb 100644 --- a/app/methods/hosted_file_methods.py +++ b/app/methods/hosted_file_methods.py @@ -144,7 +144,39 @@ def allowed_file_extension(extension: str, extension_list: list): # ### END ### API Hosted File Methods ### allowed_file_extension() ### +# ### BEGIN ### API Hosted File Methods ### lookup_file_hash() ### +# Updated 2023-09-19 +@logger_reset +def check_for_hosted_file_hash_file( + file_hash: str, + sub_dir: str, + ) -> dict|bool: + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + file_size = None + + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + log.info(f'Hosted Files Path: {hosted_files_path}') + log.debug(shutil.disk_usage(hosted_files_path)) + + hosted_files_dir_w_subdir = os.path.join(hosted_files_path, sub_dir) + path_hosted_files_dir_w_subdir = pathlib.Path(hosted_files_dir_w_subdir) + + if path_hosted_files_dir_w_subdir.exists(): pass + else: + log.warning('Hashed hosted file subdirectory was not found in the hosted files root.') + return False + + hosted_files_dir_w_subdir_filename = os.path.join(hosted_files_path, sub_dir, f'{file_hash}.file') + path_hosted_files_dir_w_subdir_filename = pathlib.Path(hosted_files_dir_w_subdir_filename) + if path_hosted_files_dir_w_subdir_filename.exists(): + file_size = os.path.getsize(path_hosted_files_dir_w_subdir_filename) + else: + log.warning('Hashed hosted file not found in the expected hosted files subdirectory.') + return False + + return {'found': True, 'file_size': file_size} # ### BEGIN ### API Hosted File Methods ### save_file() ### diff --git a/app/models/archive_content_models.py b/app/models/archive_content_models.py index b98cc11..888d59f 100644 --- a/app/models/archive_content_models.py +++ b/app/models/archive_content_models.py @@ -56,6 +56,8 @@ class Archive_Content_Base(BaseModel): # xxxx_red: str = Field(default='xxx') # xxxx_blue: str = Field(default_factory=testing) hosted_file_path: str = None # '/testing/test-test' + api_hosted_file_path_download: str = None # '/testing/test-test' + api_hosted_file_path_stream: str = None # '/testing/test-test' original_datetime: Optional[datetime.datetime] original_datetime_timezone: Optional[str] @@ -105,7 +107,7 @@ class Archive_Content_Base(BaseModel): @validator('hosted_file_path', always=True) def hosted_file_path_lookup(cls, v, values, **kwargs): - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(v) log.debug(values) @@ -116,7 +118,43 @@ class Archive_Content_Base(BaseModel): if filename := values.get('filename'): path_str = f'{path_str}?filename={filename}' - log.debug(path_str) + log.info(f'Path: {path_str}') + return path_str + log.debug('NOT Found hosted_file_id_random...') + return v + + @validator('api_hosted_file_path_download', always=True) + def api_hosted_file_path_download_lookup(cls, v, values, **kwargs): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(v) + log.debug(values) + + if hosted_file_id_random := values.get('hosted_file_id_random'): + log.debug('Found hosted_file_id_random...') + path_str = f'/hosted_file/{hosted_file_id_random}/download' + + if filename := values.get('filename'): + path_str = f'{path_str}?filename={filename}' + + log.info(f'Path: {path_str}') + return path_str + log.debug('NOT Found hosted_file_id_random...') + return v + + @validator('api_hosted_file_path_stream', always=True) + def api_hosted_file_path_stream_lookup(cls, v, values, **kwargs): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(v) + log.debug(values) + + if hosted_file_id_random := values.get('hosted_file_id_random'): + log.debug('Found hosted_file_id_random...') + path_str = f'/hosted_file/{hosted_file_id_random}/stream' + + if filename := values.get('filename'): + path_str = f'{path_str}?filename={filename}' + + log.info(f'Path: {path_str}') return path_str log.debug('NOT Found hosted_file_id_random...') return v diff --git a/app/models/hosted_file_models.py b/app/models/hosted_file_models.py index d9fd3d8..3df25ca 100644 --- a/app/models/hosted_file_models.py +++ b/app/models/hosted_file_models.py @@ -61,6 +61,15 @@ class Hosted_File_Base(BaseModel): created_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None + # Including convenience data + # This is only for convenience. Probably going to keep unless it causes a problem. + hosted_file_found_check: Optional[bool] = Field( + alias = 'found_check' + ) + hosted_file_size_check: Optional[int] = Field( # File size in bytes + alias = 'size_check' + ) + # Including other related objects hosted_file_link_list: Optional[list] # Hosted_File_Base() diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index 455b878..d77e82e 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -1,4 +1,4 @@ -import aiofiles, datetime, hashlib, mimetypes, os, pathlib, shutil, time +import aiofiles, datetime, hashlib, mimetypes, os, pathlib, random, shutil, time from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile from fastapi.responses import FileResponse, StreamingResponse # from fastapi.responses import StreamingResponse @@ -14,7 +14,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, # from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template -from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, save_file_to_hosted_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash +from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, save_file_to_hosted_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash, check_for_hosted_file_hash_file from app.models.hosted_file_models import Hosted_File_Base from app.models.response_models import Resp_Body_Base, mk_resp @@ -423,7 +423,7 @@ async def upload_files( log.debug(locals()) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING - # time.sleep(3.5) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING + time.sleep(random.choice((3.5, 4.5, 5, 6.5))) # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING # NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING NOTE: WARNING account_id_random = account_id # This is for the account random str ID @@ -894,6 +894,53 @@ async def get_hosted_file_obj( # ### END ### API Hosted File ### get_hosted_file_obj() ### + +# ### BEGIN ### API Hosted File ### get_hosted_file_obj_w_hash() ### +# Updated 2021-09-07 +@router.get('/hash/{hosted_file_hash}', response_model=Resp_Body_Base) +async def check_hosted_file_obj_w_hash( + hosted_file_hash: str = Query(..., min_length=64, max_length=64), # Expects SHA256 hash + check_for_local: Optional[bool] = True, + + commons: Common_Route_Params = Depends(common_route_params), + ): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + + if hosted_file_id := lookup_file_hash( + file_hash = hosted_file_hash, + ): + + hosted_file_dict = None + if hosted_file_obj := load_hosted_file_obj( + hosted_file_id = hosted_file_id, + ): + hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) + else: + # NOTE: This should not ever happen if the ID was already found. + return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash was found in the database, but something went wrong while loading the details from the database: Hosted File ID: {hosted_file_id}; Hosted File Hash: {hosted_file_hash}', response=commons.response) # Not Found + + if check_for_local: + if check_for_hosted_file_hash_file_results := check_for_hosted_file_hash_file( + file_hash = hosted_file_hash, + sub_dir = hosted_file_obj.subdirectory_path, + ): + hosted_file_dict = hosted_file_obj.dict(by_alias=commons.by_alias, exclude_unset=commons.exclude_unset) + hosted_file_dict['hosted_file_found_check'] = True + hosted_file_dict['hosted_file_size_check'] = check_for_hosted_file_hash_file_results['file_size'] # File size in bytes + + else: + return mk_resp(data=False, status_code=500, response=commons.response) # Bad Request + + else: + return mk_resp(data=False, status_code=404, status_message=f'Hosted file with hash not found in the database: {hosted_file_hash}', response=commons.response) # Not Found + + return mk_resp(data=hosted_file_dict, response=commons.response) + #return mk_resp(data=hosted_file_obj) +# ### END ### API Hosted File ### get_hosted_file_obj() ### + + # ### BEGIN ### API Hosted File ### download_tmp() ### # Updated 2023-04-05 @router.get('/tmp/{subdirectory}/{filename}/download', response_model=Resp_Body_Base)