Work on hosted files. Now with basic hosted directory check and clean up.
This commit is contained in:
Scott Idem
@@ -78,6 +78,44 @@ def load_hosted_file_obj(
|
||||
# ### END ### API Hosted File Methods ### load_hosted_file_obj() ###
|
||||
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### lookup_file_hash() ###
|
||||
# Updated 2022-08-09
|
||||
@logger_reset
|
||||
def lookup_file_hash(
|
||||
file_hash: str,
|
||||
) -> Hosted_File_Base|dict|bool:
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
|
||||
sql = f"""
|
||||
SELECT id AS 'hosted_file_id', id_random AS 'hosted_file_id_random'
|
||||
FROM hosted_file
|
||||
WHERE hosted_file.hash_sha256 = :hash_sha256
|
||||
"""
|
||||
log.debug(sql)
|
||||
|
||||
hosted_file_data = {}
|
||||
hosted_file_data['hash_sha256'] = file_hash
|
||||
log.debug(hosted_file_data)
|
||||
|
||||
if hosted_file_select_result := sql_select(sql=sql, data=hosted_file_data):
|
||||
hosted_file_id = hosted_file_select_result.get('hosted_file_id')
|
||||
hosted_file_id_random = hosted_file_select_result.get('hosted_file_id_random')
|
||||
log.info(f'Selected Hosted File record. Hosted File ID: {hosted_file_id}')
|
||||
return hosted_file_id
|
||||
elif hosted_file_select_result is None:
|
||||
log.warning(f'Hosted File record was not found. SHA 256 Hash: {file_hash}')
|
||||
return None
|
||||
# pass
|
||||
else:
|
||||
log.error(f'Something went wrong while trying to select the hosted file record. SHA 256 Hash: {file_hash}')
|
||||
return False
|
||||
# ### END ### API Hosted File Methods ### lookup_file_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
|
||||
@logger_reset
|
||||
async def get_file_object_hash(file_object:File):
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import datetime, pytz
|
||||
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.db_sql import redis_lookup_id_random
|
||||
from app.lib_general import log, logging
|
||||
|
||||
from .common_field_schema import base_fields, default_num_bytes
|
||||
from app.models.common_field_schema import base_fields, default_num_bytes
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Link Models ### Hosted_File_Link_Base() ###
|
||||
@@ -13,10 +14,10 @@ class Hosted_File_Link_Base(BaseModel):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
id_random: Optional[str] = Field(
|
||||
**base_fields['hosted_file_link_id_random'],
|
||||
alias = 'hosted_file_link_id_random',
|
||||
)
|
||||
# id_random: Optional[str] = Field(
|
||||
# **base_fields['hosted_file_link_id_random'],
|
||||
# alias = 'hosted_file_link_id_random',
|
||||
# )
|
||||
id: Optional[int] = Field(
|
||||
#alias = 'hosted_file_link_id'
|
||||
)
|
||||
@@ -39,32 +40,11 @@ class Hosted_File_Link_Base(BaseModel):
|
||||
|
||||
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
|
||||
|
||||
#@validator('hosted_file_link_id_random', always=True)
|
||||
def hosted_file_link_id_random_copy(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['id_random']:
|
||||
return values['id_random']
|
||||
return None
|
||||
|
||||
@validator('id', always=True)
|
||||
def hosted_file_link_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['id_random']:
|
||||
log.debug(values['id_random'])
|
||||
return redis_lookup_id_random(record_id_random=values['id_random'], table_name='hosted_file_link')
|
||||
return None
|
||||
|
||||
@validator('account_id', always=True)
|
||||
def account_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['account_id_random']:
|
||||
return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account')
|
||||
if isinstance(v, int) and v > 0: return v
|
||||
elif id_random := values.get('account_id_random'):
|
||||
return redis_lookup_id_random(record_id_random=id_random, table_name='account')
|
||||
return None
|
||||
|
||||
@validator('link_to_id', always=True)
|
||||
@@ -78,5 +58,6 @@ class Hosted_File_Link_Base(BaseModel):
|
||||
|
||||
class Config:
|
||||
underscore_attrs_are_private = True
|
||||
allow_population_by_field_name = True
|
||||
fields = base_fields
|
||||
# ### END ### API Hosted File Link Models ### Hosted_File_Link_Base() ###
|
||||
|
||||
@@ -10,7 +10,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
|
||||
|
||||
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
|
||||
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list
|
||||
from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
|
||||
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
@@ -19,6 +19,118 @@ from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File ### directory_check() ###
|
||||
# Updated 2022-08-09
|
||||
@router.get('/directory_check', response_model=Resp_Body_Base)
|
||||
async def directory_check(
|
||||
rm_orphan: bool = False,
|
||||
|
||||
commons: Common_Route_Params = Depends(common_route_params),
|
||||
):
|
||||
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# print('HERE HERE HERE')
|
||||
|
||||
# return mk_resp(data=True, response=commons.response, status_message='HERE HERE HERE The hosted file directory check.')
|
||||
|
||||
|
||||
# ### Orphan file: ### Delete file from server
|
||||
hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
# hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||
log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||
|
||||
full_directory_path = hosted_files_path
|
||||
log.debug(full_directory_path)
|
||||
# file_path_w_subdir = os.path(full_directory_path)
|
||||
# log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
|
||||
|
||||
if os.path.isdir(full_directory_path):
|
||||
log.info('Path exists! Going to get a list of files...')
|
||||
directory_list = os.listdir(full_directory_path)
|
||||
|
||||
result_list = []
|
||||
for directory_item in directory_list:
|
||||
file_path_w_item = os.path.join(full_directory_path, directory_item)
|
||||
# log.info(f'Full file path with directory item: {file_path_w_item}')
|
||||
log.info(f'Checking directory item: {directory_item}')
|
||||
if os.path.isfile(file_path_w_item):
|
||||
# log.debug(f'File: {directory_item}')
|
||||
# result_list.append(file_path_w_item)
|
||||
|
||||
if '.file' in directory_item: pass
|
||||
else:
|
||||
log.warning(f'Not a hashed file! File: {directory_item}')
|
||||
continue
|
||||
|
||||
if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')):
|
||||
# log.info('DB record found')
|
||||
# result_list.append(file_path_w_item)
|
||||
pass
|
||||
else:
|
||||
log.warning(f'Hosted File record not found!!! File: {directory_item}')
|
||||
result_list.append(file_path_w_item)
|
||||
if rm_orphan:
|
||||
log.info('Going remove the hosted file from server...')
|
||||
|
||||
try:
|
||||
# log.warning('DELETE')
|
||||
pathlib.Path(file_path_w_item).unlink()
|
||||
# continue
|
||||
except OSError as e:
|
||||
log.error("Error: %s : %s" % (file_path, e.strerror))
|
||||
# return False
|
||||
continue
|
||||
else:
|
||||
# log.debug(f'Directory: {directory_item}')
|
||||
# pass
|
||||
log.info('Subdirectory Path exists! Going to get a list of files...')
|
||||
full_subdirectory_path = os.path.join(full_directory_path, directory_item)
|
||||
subdirectory_list = os.listdir(full_subdirectory_path)
|
||||
|
||||
subdirectory_result_list = []
|
||||
for subdirectory_item in subdirectory_list:
|
||||
file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item)
|
||||
# log.info(f'Full file path with directory item: {file_path_w_item}')
|
||||
log.info(f'Checking subdirectory item: {subdirectory_item}')
|
||||
if os.path.isfile(file_path_w_item):
|
||||
# log.debug(f'File: {subdirectory_item}')
|
||||
# subdirectory_result_list.append(file_path_w_item)
|
||||
|
||||
if '.file' in subdirectory_item: pass
|
||||
else:
|
||||
log.warning(f'Not a hashed file! File: {subdirectory_item}')
|
||||
continue
|
||||
|
||||
if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')):
|
||||
# log.info('DB record found')
|
||||
# subdirectory_result_list.append(file_path_w_item)
|
||||
pass
|
||||
else:
|
||||
log.warning(f'Hosted File record not found!!! File: {subdirectory_item}')
|
||||
result_list.append(file_path_w_item)
|
||||
if rm_orphan:
|
||||
log.info('Going remove the hosted file from server...')
|
||||
|
||||
try:
|
||||
# log.warning('DELETE')
|
||||
pathlib.Path(file_path_w_item).unlink()
|
||||
# continue
|
||||
except OSError as e:
|
||||
log.error("Error: %s : %s" % (file_path, e.strerror))
|
||||
# return False
|
||||
continue
|
||||
else:
|
||||
log.warning(f'Subdirectory: {subdirectory_item}')
|
||||
pass
|
||||
|
||||
return mk_resp(data=result_list, response=commons.response, status_message='The hosted file directory check.')
|
||||
else:
|
||||
log.warning(f'The hosted file directory was not found on the server.')
|
||||
mk_resp(data=False, status_code=500, response=commons.response, status_message='Something may have gone wrong while trying to check the hosted file directory.') # Internal Server Error
|
||||
# ### END ### API Hosted File ### directory_check() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File ### download_hosted_file() ###
|
||||
# Updated 2022-08-08
|
||||
@router.get('/{hosted_file_id}/download', response_model=Resp_Body_Base)
|
||||
@@ -497,7 +609,7 @@ async def test_upload_files(
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File ### delete_hosted_file() ###
|
||||
# Updated 2022-08-08
|
||||
# Updated 2022-08-09
|
||||
@router.delete('/{hosted_file_id}', response_model=Resp_Body_Base)
|
||||
async def delete_hosted_file(
|
||||
hosted_file_id: str = Query(..., min_length=11, max_length=22),
|
||||
@@ -516,6 +628,7 @@ async def delete_hosted_file(
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
|
||||
|
||||
# ### SECTION ### Handle the deletion of records and file
|
||||
if hosted_file_delete_result := handle_delete_hosted_file(account_id=commons.x_account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan):
|
||||
return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
|
||||
elif hosted_file_delete_result is None:
|
||||
@@ -524,116 +637,6 @@ async def delete_hosted_file(
|
||||
else:
|
||||
log.error(f'Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.')
|
||||
return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') # Bad Request
|
||||
|
||||
|
||||
|
||||
# # ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE
|
||||
# # NOTE: If link_to_type and link_to_id passed then try and remove that link record first.
|
||||
|
||||
# # if hosted_file_result := delete_hosted_file_link(
|
||||
# # account_id = commons.x_account_id,
|
||||
# # hosted_file_id = hosted_file_id,
|
||||
|
||||
# # link_to_type = link_to_type,
|
||||
# # link_to_id = link_to_id,
|
||||
|
||||
# # rm_orphan = rm_orphan,
|
||||
# # ):
|
||||
# # pass
|
||||
# # elif hosted_file_result is None:
|
||||
# # log.error('The hosted file link record was not found and may have already been deleted.')
|
||||
# # return mk_resp(data=False, status_code=404, response=commons.response) # Not Found
|
||||
# # else:
|
||||
# # log.error('Something went wrong while trying to delete the hosted file link record.')
|
||||
# # return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
|
||||
|
||||
# # ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE
|
||||
# # NOTE: If not rm_orphan then do nothing else.
|
||||
# # NOTE: If rm_orphan then get list of links for file.
|
||||
# # NOTE: If 0 links result then delete the hosted_file record and file on the server.
|
||||
# # NOTE: If >0 links result then do nothing else.
|
||||
|
||||
# # NOTE: Don't check or remove orphan
|
||||
# if not rm_orphan:
|
||||
# log.info('Remove hosted file link. No orphan check.')
|
||||
# return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
|
||||
|
||||
|
||||
# if hosted_file_delete_result := handle_delete_hosted_file(account_id=account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan):
|
||||
# return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
|
||||
# else:
|
||||
# pass
|
||||
|
||||
# # # NOTE: Check and remove orphan
|
||||
# # if hosted_file_link_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id):
|
||||
# # hosted_file_link_result_list = []
|
||||
# # for hosted_file_link_rec in hosted_file_link_rec_list_result:
|
||||
# # hosted_file_link_result_list.append(hosted_file_link_rec)
|
||||
|
||||
# # log.debug(hosted_file_link_result_list)
|
||||
# # hosted_file_list = hosted_file_link_result_list
|
||||
# # # NOT safe to delete the hosted_file record and file from server!!!
|
||||
# # # STOP!
|
||||
# # log.info('Remove hosted file link. Not an orphan file.')
|
||||
# # return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
|
||||
# # elif isinstance(hosted_file_link_rec_list_result, list) or hosted_file_link_rec_list_result is None:
|
||||
# # hosted_file_list = []
|
||||
# # # Safe to delete the hosted_file record and file from server???
|
||||
# # # CONTINUE
|
||||
# # delete_hosted_file(hosted_file_id=hosted_file_id)
|
||||
# # else:
|
||||
# # hosted_file_list = False
|
||||
# # # Safe to delete the hosted_file record and file from server???
|
||||
# # # CONTINUE
|
||||
|
||||
|
||||
# # hosted_files_path = settings.FILES_PATH['hosted_files_root']
|
||||
# hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
|
||||
# log.info(f'Hosted Files Path: {hosted_files_path}')
|
||||
|
||||
# if hosted_file_obj := load_hosted_file_obj(
|
||||
# hosted_file_id = hosted_file_id,
|
||||
# # inc_hosted_file = True,
|
||||
# inc_hosted_file_link_list = rm_orphan, # if rm_orphan then need to include hosted_file_link_list
|
||||
# ):
|
||||
# pass
|
||||
# else:
|
||||
# return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
|
||||
|
||||
# # if not filename:
|
||||
# # filename = hosted_file_obj.filename
|
||||
# # log.info(f'Filename: {filename}')
|
||||
# dir_path = hosted_file_obj.directory_path
|
||||
# subdir_path = hosted_file_obj.subdirectory_path
|
||||
# hash_sha256 = hosted_file_obj.hash_sha256
|
||||
# hash_filename = hash_sha256+'.file'
|
||||
|
||||
# if subdir_path:
|
||||
# full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
|
||||
# else:
|
||||
# full_subdirectory_path = hosted_files_path
|
||||
# log.debug(full_subdirectory_path)
|
||||
# # pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
|
||||
# file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
|
||||
# log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
|
||||
|
||||
# if os.path.exists(file_path_w_subdir):
|
||||
# log.info('File exists!')
|
||||
# if rm_orphan:
|
||||
# log.info('Going remove the file if it is an orphan...')
|
||||
# # try:
|
||||
# # pathlib.Path(file_path_w_subdir).unlink()
|
||||
# # except OSError as e:
|
||||
# # log.error("Error: %s : %s" % (file_path, e.strerror))
|
||||
# # return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something went wrong while trying to delete the hosted file from the server.') # Bad Request
|
||||
# return mk_resp(data=True, response=commons.response, status_message='The orphan hosted file was deleted from the server.')
|
||||
# else:
|
||||
# log.info('Remove hosted file link. No orphan check.')
|
||||
# return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
|
||||
|
||||
# else:
|
||||
# log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
|
||||
# return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
|
||||
# ### END ### API Hosted File ### download_hosted_file() ###
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user