From 8d502a9fd07dfe51c65532e18fb9cc9f4c4c6427 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 9 Aug 2022 17:43:30 -0400 Subject: [PATCH] Work on hosted files. Now with basic hosted directory check and clean up. --- app/methods/hosted_file_methods.py | 38 +++++ app/models/hosted_file_link_models.py | 39 ++--- app/routers/hosted_file.py | 227 +++++++++++++------------- 3 files changed, 163 insertions(+), 141 deletions(-) diff --git a/app/methods/hosted_file_methods.py b/app/methods/hosted_file_methods.py index 6a7f126..ab52b70 100644 --- a/app/methods/hosted_file_methods.py +++ b/app/methods/hosted_file_methods.py @@ -78,6 +78,44 @@ def load_hosted_file_obj( # ### END ### API Hosted File Methods ### load_hosted_file_obj() ### + + +# ### BEGIN ### API Hosted File Methods ### lookup_file_hash() ### +# Updated 2022-08-09 +@logger_reset +def lookup_file_hash( + file_hash: str, + ) -> Hosted_File_Base|dict|bool: + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + + sql = f""" + SELECT id AS 'hosted_file_id', id_random AS 'hosted_file_id_random' + FROM hosted_file + WHERE hosted_file.hash_sha256 = :hash_sha256 + """ + log.debug(sql) + + hosted_file_data = {} + hosted_file_data['hash_sha256'] = file_hash + log.debug(hosted_file_data) + + if hosted_file_select_result := sql_select(sql=sql, data=hosted_file_data): + hosted_file_id = hosted_file_select_result.get('hosted_file_id') + hosted_file_id_random = hosted_file_select_result.get('hosted_file_id_random') + log.info(f'Selected Hosted File record. Hosted File ID: {hosted_file_id}') + return hosted_file_id + elif hosted_file_select_result is None: + log.warning(f'Hosted File record was not found. SHA 256 Hash: {file_hash}') + return None + # pass + else: + log.error(f'Something went wrong while trying to select the hosted file record. SHA 256 Hash: {file_hash}') + return False +# ### END ### API Hosted File Methods ### lookup_file_hash() ### + + # ### BEGIN ### API Hosted File Route ### get_file_object_hash() ### @logger_reset async def get_file_object_hash(file_object:File): diff --git a/app/models/hosted_file_link_models.py b/app/models/hosted_file_link_models.py index e2c1acc..2a0a91b 100644 --- a/app/models/hosted_file_link_models.py +++ b/app/models/hosted_file_link_models.py @@ -1,11 +1,12 @@ import datetime, pytz + from typing import Dict, List, Optional, Set, Union from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator from app.db_sql import redis_lookup_id_random from app.lib_general import log, logging -from .common_field_schema import base_fields, default_num_bytes +from app.models.common_field_schema import base_fields, default_num_bytes # ### BEGIN ### API Hosted File Link Models ### Hosted_File_Link_Base() ### @@ -13,10 +14,10 @@ class Hosted_File_Link_Base(BaseModel): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - id_random: Optional[str] = Field( - **base_fields['hosted_file_link_id_random'], - alias = 'hosted_file_link_id_random', - ) + # id_random: Optional[str] = Field( + # **base_fields['hosted_file_link_id_random'], + # alias = 'hosted_file_link_id_random', + # ) id: Optional[int] = Field( #alias = 'hosted_file_link_id' ) @@ -39,32 +40,11 @@ class Hosted_File_Link_Base(BaseModel): _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) - #@validator('hosted_file_link_id_random', always=True) - def hosted_file_link_id_random_copy(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['id_random']: - return values['id_random'] - return None - - @validator('id', always=True) - def hosted_file_link_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['id_random']: - log.debug(values['id_random']) - return redis_lookup_id_random(record_id_random=values['id_random'], table_name='hosted_file_link') - return None - @validator('account_id', always=True) def account_id_lookup(cls, v, values, **kwargs): - log.setLevel(logging.WARNING) - log.debug(locals()) - - if values['account_id_random']: - return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account') + if isinstance(v, int) and v > 0: return v + elif id_random := values.get('account_id_random'): + return redis_lookup_id_random(record_id_random=id_random, table_name='account') return None @validator('link_to_id', always=True) @@ -78,5 +58,6 @@ class Hosted_File_Link_Base(BaseModel): class Config: underscore_attrs_are_private = True + allow_population_by_field_name = True fields = base_fields # ### END ### API Hosted File Link Models ### Hosted_File_Link_Base() ### diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index dca0e0c..7815afc 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -10,7 +10,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, # from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template -from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list +from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash from app.models.hosted_file_models import Hosted_File_Base from app.models.response_models import Resp_Body_Base, mk_resp @@ -19,6 +19,118 @@ from app.models.response_models import Resp_Body_Base, mk_resp router = APIRouter() +# ### BEGIN ### API Hosted File ### directory_check() ### +# Updated 2022-08-09 +@router.get('/directory_check', response_model=Resp_Body_Base) +async def directory_check( + rm_orphan: bool = False, + + commons: Common_Route_Params = Depends(common_route_params), + ): + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + # print('HERE HERE HERE') + + # return mk_resp(data=True, response=commons.response, status_message='HERE HERE HERE The hosted file directory check.') + + + # ### Orphan file: ### Delete file from server + hosted_files_path = settings.FILES_PATH['hosted_files_root'] + # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' + log.info(f'Hosted Files Path: {hosted_files_path}') + + full_directory_path = hosted_files_path + log.debug(full_directory_path) + # file_path_w_subdir = os.path(full_directory_path) + # log.info(f'Full file path with subdirectory: {file_path_w_subdir}') + + if os.path.isdir(full_directory_path): + log.info('Path exists! Going to get a list of files...') + directory_list = os.listdir(full_directory_path) + + result_list = [] + for directory_item in directory_list: + file_path_w_item = os.path.join(full_directory_path, directory_item) + # log.info(f'Full file path with directory item: {file_path_w_item}') + log.info(f'Checking directory item: {directory_item}') + if os.path.isfile(file_path_w_item): + # log.debug(f'File: {directory_item}') + # result_list.append(file_path_w_item) + + if '.file' in directory_item: pass + else: + log.warning(f'Not a hashed file! File: {directory_item}') + continue + + if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')): + # log.info('DB record found') + # result_list.append(file_path_w_item) + pass + else: + log.warning(f'Hosted File record not found!!! File: {directory_item}') + result_list.append(file_path_w_item) + if rm_orphan: + log.info('Going remove the hosted file from server...') + + try: + # log.warning('DELETE') + pathlib.Path(file_path_w_item).unlink() + # continue + except OSError as e: + log.error("Error: %s : %s" % (file_path, e.strerror)) + # return False + continue + else: + # log.debug(f'Directory: {directory_item}') + # pass + log.info('Subdirectory Path exists! Going to get a list of files...') + full_subdirectory_path = os.path.join(full_directory_path, directory_item) + subdirectory_list = os.listdir(full_subdirectory_path) + + subdirectory_result_list = [] + for subdirectory_item in subdirectory_list: + file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item) + # log.info(f'Full file path with directory item: {file_path_w_item}') + log.info(f'Checking subdirectory item: {subdirectory_item}') + if os.path.isfile(file_path_w_item): + # log.debug(f'File: {subdirectory_item}') + # subdirectory_result_list.append(file_path_w_item) + + if '.file' in subdirectory_item: pass + else: + log.warning(f'Not a hashed file! File: {subdirectory_item}') + continue + + if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')): + # log.info('DB record found') + # subdirectory_result_list.append(file_path_w_item) + pass + else: + log.warning(f'Hosted File record not found!!! File: {subdirectory_item}') + result_list.append(file_path_w_item) + if rm_orphan: + log.info('Going remove the hosted file from server...') + + try: + # log.warning('DELETE') + pathlib.Path(file_path_w_item).unlink() + # continue + except OSError as e: + log.error("Error: %s : %s" % (file_path, e.strerror)) + # return False + continue + else: + log.warning(f'Subdirectory: {subdirectory_item}') + pass + + return mk_resp(data=result_list, response=commons.response, status_message='The hosted file directory check.') + else: + log.warning(f'The hosted file directory was not found on the server.') + mk_resp(data=False, status_code=500, response=commons.response, status_message='Something may have gone wrong while trying to check the hosted file directory.') # Internal Server Error +# ### END ### API Hosted File ### directory_check() ### + + # ### BEGIN ### API Hosted File ### download_hosted_file() ### # Updated 2022-08-08 @router.get('/{hosted_file_id}/download', response_model=Resp_Body_Base) @@ -497,7 +609,7 @@ async def test_upload_files( # ### BEGIN ### API Hosted File ### delete_hosted_file() ### -# Updated 2022-08-08 +# Updated 2022-08-09 @router.delete('/{hosted_file_id}', response_model=Resp_Body_Base) async def delete_hosted_file( hosted_file_id: str = Query(..., min_length=11, max_length=22), @@ -516,6 +628,7 @@ async def delete_hosted_file( if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.') + # ### SECTION ### Handle the deletion of records and file if hosted_file_delete_result := handle_delete_hosted_file(account_id=commons.x_account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan): return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.') elif hosted_file_delete_result is None: @@ -524,116 +637,6 @@ async def delete_hosted_file( else: log.error(f'Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') # Bad Request - - - - # # ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE - # # NOTE: If link_to_type and link_to_id passed then try and remove that link record first. - - # # if hosted_file_result := delete_hosted_file_link( - # # account_id = commons.x_account_id, - # # hosted_file_id = hosted_file_id, - - # # link_to_type = link_to_type, - # # link_to_id = link_to_id, - - # # rm_orphan = rm_orphan, - # # ): - # # pass - # # elif hosted_file_result is None: - # # log.error('The hosted file link record was not found and may have already been deleted.') - # # return mk_resp(data=False, status_code=404, response=commons.response) # Not Found - # # else: - # # log.error('Something went wrong while trying to delete the hosted file link record.') - # # return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request - - # # ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE - # # NOTE: If not rm_orphan then do nothing else. - # # NOTE: If rm_orphan then get list of links for file. - # # NOTE: If 0 links result then delete the hosted_file record and file on the server. - # # NOTE: If >0 links result then do nothing else. - - # # NOTE: Don't check or remove orphan - # if not rm_orphan: - # log.info('Remove hosted file link. No orphan check.') - # return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.') - - - # if hosted_file_delete_result := handle_delete_hosted_file(account_id=account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan): - # return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.') - # else: - # pass - - # # # NOTE: Check and remove orphan - # # if hosted_file_link_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id): - # # hosted_file_link_result_list = [] - # # for hosted_file_link_rec in hosted_file_link_rec_list_result: - # # hosted_file_link_result_list.append(hosted_file_link_rec) - - # # log.debug(hosted_file_link_result_list) - # # hosted_file_list = hosted_file_link_result_list - # # # NOT safe to delete the hosted_file record and file from server!!! - # # # STOP! - # # log.info('Remove hosted file link. Not an orphan file.') - # # return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.') - # # elif isinstance(hosted_file_link_rec_list_result, list) or hosted_file_link_rec_list_result is None: - # # hosted_file_list = [] - # # # Safe to delete the hosted_file record and file from server??? - # # # CONTINUE - # # delete_hosted_file(hosted_file_id=hosted_file_id) - # # else: - # # hosted_file_list = False - # # # Safe to delete the hosted_file record and file from server??? - # # # CONTINUE - - - # # hosted_files_path = settings.FILES_PATH['hosted_files_root'] - # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' - # log.info(f'Hosted Files Path: {hosted_files_path}') - - # if hosted_file_obj := load_hosted_file_obj( - # hosted_file_id = hosted_file_id, - # # inc_hosted_file = True, - # inc_hosted_file_link_list = rm_orphan, # if rm_orphan then need to include hosted_file_link_list - # ): - # pass - # else: - # return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request - - # # if not filename: - # # filename = hosted_file_obj.filename - # # log.info(f'Filename: {filename}') - # dir_path = hosted_file_obj.directory_path - # subdir_path = hosted_file_obj.subdirectory_path - # hash_sha256 = hosted_file_obj.hash_sha256 - # hash_filename = hash_sha256+'.file' - - # if subdir_path: - # full_subdirectory_path = os.path.join(hosted_files_path, subdir_path) - # else: - # full_subdirectory_path = hosted_files_path - # log.debug(full_subdirectory_path) - # # pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True) - # file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename) - # log.info(f'Full file path with subdirectory: {file_path_w_subdir}') - - # if os.path.exists(file_path_w_subdir): - # log.info('File exists!') - # if rm_orphan: - # log.info('Going remove the file if it is an orphan...') - # # try: - # # pathlib.Path(file_path_w_subdir).unlink() - # # except OSError as e: - # # log.error("Error: %s : %s" % (file_path, e.strerror)) - # # return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something went wrong while trying to delete the hosted file from the server.') # Bad Request - # return mk_resp(data=True, response=commons.response, status_message='The orphan hosted file was deleted from the server.') - # else: - # log.info('Remove hosted file link. No orphan check.') - # return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.') - - # else: - # log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}') - # return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request # ### END ### API Hosted File ### download_hosted_file() ###