Work on hosted files. Now with basic hosted directory check and clean up.

app/methods/hosted_file_methods.py

@@ -78,6 +78,44 @@ def load_hosted_file_obj(
 # ### END ### API Hosted File Methods ### load_hosted_file_obj() ###
 
 
+
+
+# ### BEGIN ### API Hosted File Methods ### lookup_file_hash() ###
+# Updated 2022-08-09
+@logger_reset
+def lookup_file_hash(
+        file_hash: str,
+        ) -> Hosted_File_Base|dict|bool:
+    log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+
+    sql = f"""
+        SELECT id AS 'hosted_file_id', id_random AS 'hosted_file_id_random'
+        FROM hosted_file
+        WHERE hosted_file.hash_sha256 = :hash_sha256
+        """
+    log.debug(sql)
+
+    hosted_file_data = {}
+    hosted_file_data['hash_sha256'] = file_hash
+    log.debug(hosted_file_data)
+
+    if hosted_file_select_result := sql_select(sql=sql, data=hosted_file_data):
+        hosted_file_id = hosted_file_select_result.get('hosted_file_id')
+        hosted_file_id_random = hosted_file_select_result.get('hosted_file_id_random')
+        log.info(f'Selected Hosted File record. Hosted File ID: {hosted_file_id}')
+        return hosted_file_id
+    elif hosted_file_select_result is None:
+        log.warning(f'Hosted File record was not found. SHA 256 Hash: {file_hash}')
+        return None
+        # pass
+    else:
+        log.error(f'Something went wrong while trying to select the hosted file record. SHA 256 Hash: {file_hash}')
+        return False
+# ### END ### API Hosted File Methods ### lookup_file_hash() ###
+
+
 # ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
 @logger_reset
 async def get_file_object_hash(file_object:File):

app/models/hosted_file_link_models.py

@@ -1,11 +1,12 @@
 import datetime, pytz
+
 from typing import Dict, List, Optional, Set, Union
 from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
 
 from app.db_sql import redis_lookup_id_random
 from app.lib_general import log, logging
 
-from .common_field_schema import base_fields, default_num_bytes
+from app.models.common_field_schema import base_fields, default_num_bytes
 
 
 # ### BEGIN ### API Hosted File Link Models ### Hosted_File_Link_Base() ###
@@ -13,10 +14,10 @@ class Hosted_File_Link_Base(BaseModel):
     log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(locals())
 
-    id_random: Optional[str] = Field(
+    # id_random: Optional[str] = Field(
-        **base_fields['hosted_file_link_id_random'],
+    #     **base_fields['hosted_file_link_id_random'],
-        alias = 'hosted_file_link_id_random',
+    #     alias = 'hosted_file_link_id_random',
-    )
+    # )
     id: Optional[int] = Field(
         #alias = 'hosted_file_link_id'
     )
@@ -39,32 +40,11 @@ class Hosted_File_Link_Base(BaseModel):
 
     _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
 
-    #@validator('hosted_file_link_id_random', always=True)
-    def hosted_file_link_id_random_copy(cls, v, values, **kwargs):
-        log.setLevel(logging.WARNING)
-        log.debug(locals())
-
-        if values['id_random']:
-            return values['id_random']
-        return None
-
-    @validator('id', always=True)
-    def hosted_file_link_id_lookup(cls, v, values, **kwargs):
-        log.setLevel(logging.WARNING)
-        log.debug(locals())
-
-        if values['id_random']:
-            log.debug(values['id_random'])
-            return redis_lookup_id_random(record_id_random=values['id_random'], table_name='hosted_file_link')
-        return None
-
     @validator('account_id', always=True)
     def account_id_lookup(cls, v, values, **kwargs):
-        log.setLevel(logging.WARNING)
+        if isinstance(v, int) and v > 0: return v
-        log.debug(locals())
+        elif id_random := values.get('account_id_random'):
-
+            return redis_lookup_id_random(record_id_random=id_random, table_name='account')
-        if values['account_id_random']:
-            return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account')
         return None
 
     @validator('link_to_id', always=True)
@@ -78,5 +58,6 @@ class Hosted_File_Link_Base(BaseModel):
 
     class Config:
         underscore_attrs_are_private = True
+        allow_population_by_field_name = True
         fields = base_fields
 # ### END ### API Hosted File Link Models ### Hosted_File_Link_Base() ###

app/routers/hosted_file.py

@@ -10,7 +10,7 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
 
 # from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
 
-from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list
+from app.methods.hosted_file_methods import create_hosted_file_obj, handle_delete_hosted_file, load_hosted_file_obj, save_file, create_hosted_file_link, delete_hosted_file_link, get_hosted_file_link_rec_list, lookup_file_hash
 
 from app.models.hosted_file_models import Hosted_File_Base
 from app.models.response_models import Resp_Body_Base, mk_resp
@@ -19,6 +19,118 @@ from app.models.response_models import Resp_Body_Base, mk_resp
 router = APIRouter()
 
 
+# ### BEGIN ### API Hosted File ### directory_check() ###
+# Updated 2022-08-09
+@router.get('/directory_check', response_model=Resp_Body_Base)
+async def directory_check(
+        rm_orphan: bool = False,
+
+        commons: Common_Route_Params = Depends(common_route_params),
+        ):
+    log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    # print('HERE HERE HERE')
+
+    # return mk_resp(data=True, response=commons.response, status_message='HERE HERE HERE The hosted file directory check.')
+
+
+    # ### Orphan file: ### Delete file from server
+    hosted_files_path = settings.FILES_PATH['hosted_files_root']
+    # hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
+    log.info(f'Hosted Files Path: {hosted_files_path}')
+
+    full_directory_path = hosted_files_path
+    log.debug(full_directory_path)
+    # file_path_w_subdir = os.path(full_directory_path)
+    # log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
+
+    if os.path.isdir(full_directory_path):
+        log.info('Path exists! Going to get a list of files...')
+        directory_list = os.listdir(full_directory_path)
+
+        result_list = []
+        for directory_item in directory_list:
+            file_path_w_item = os.path.join(full_directory_path, directory_item)
+            # log.info(f'Full file path with directory item: {file_path_w_item}')
+            log.info(f'Checking directory item: {directory_item}')
+            if os.path.isfile(file_path_w_item):
+                # log.debug(f'File: {directory_item}')
+                # result_list.append(file_path_w_item)
+
+                if '.file' in directory_item: pass
+                else:
+                    log.warning(f'Not a hashed file! File: {directory_item}')
+                    continue
+
+                if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')):
+                    # log.info('DB record found')
+                    # result_list.append(file_path_w_item)
+                    pass
+                else:
+                    log.warning(f'Hosted File record not found!!! File: {directory_item}')
+                    result_list.append(file_path_w_item)
+                    if rm_orphan:
+                        log.info('Going remove the hosted file from server...')
+
+                        try:
+                            # log.warning('DELETE')
+                            pathlib.Path(file_path_w_item).unlink()
+                            # continue
+                        except OSError as e:
+                            log.error("Error: %s : %s" % (file_path, e.strerror))
+                            # return False
+                            continue
+            else:
+                # log.debug(f'Directory: {directory_item}')
+                # pass
+                log.info('Subdirectory Path exists! Going to get a list of files...')
+                full_subdirectory_path = os.path.join(full_directory_path, directory_item)
+                subdirectory_list = os.listdir(full_subdirectory_path)
+
+                subdirectory_result_list = []
+                for subdirectory_item in subdirectory_list:
+                    file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item)
+                    # log.info(f'Full file path with directory item: {file_path_w_item}')
+                    log.info(f'Checking subdirectory item: {subdirectory_item}')
+                    if os.path.isfile(file_path_w_item):
+                        # log.debug(f'File: {subdirectory_item}')
+                        # subdirectory_result_list.append(file_path_w_item)
+
+                        if '.file' in subdirectory_item: pass
+                        else:
+                            log.warning(f'Not a hashed file! File: {subdirectory_item}')
+                            continue
+
+                        if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')):
+                            # log.info('DB record found')
+                            # subdirectory_result_list.append(file_path_w_item)
+                            pass
+                        else:
+                            log.warning(f'Hosted File record not found!!! File: {subdirectory_item}')
+                            result_list.append(file_path_w_item)
+                            if rm_orphan:
+                                log.info('Going remove the hosted file from server...')
+
+                                try:
+                                    # log.warning('DELETE')
+                                    pathlib.Path(file_path_w_item).unlink()
+                                    # continue
+                                except OSError as e:
+                                    log.error("Error: %s : %s" % (file_path, e.strerror))
+                                    # return False
+                                    continue
+                    else:
+                        log.warning(f'Subdirectory: {subdirectory_item}')
+                        pass
+
+        return mk_resp(data=result_list, response=commons.response, status_message='The hosted file directory check.')
+    else:
+        log.warning(f'The hosted file directory was not found on the server.')
+        mk_resp(data=False, status_code=500, response=commons.response, status_message='Something may have gone wrong while trying to check the hosted file directory.') # Internal Server Error
+# ### END ### API Hosted File ### directory_check() ###
+
+
 # ### BEGIN ### API Hosted File ### download_hosted_file() ###
 # Updated 2022-08-08
 @router.get('/{hosted_file_id}/download', response_model=Resp_Body_Base)
@@ -497,7 +609,7 @@ async def test_upload_files(
 
 
 # ### BEGIN ### API Hosted File ### delete_hosted_file() ###
-# Updated 2022-08-08
+# Updated 2022-08-09
 @router.delete('/{hosted_file_id}', response_model=Resp_Body_Base)
 async def delete_hosted_file(
         hosted_file_id: str = Query(..., min_length=11, max_length=22),
@@ -516,6 +628,7 @@ async def delete_hosted_file(
     if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
     else: return mk_resp(data=None, status_code=404, response=commons.response, status_message='The hosted_file ID was invalid or not found.')
 
+    # ### SECTION ### Handle the deletion of records and file
     if hosted_file_delete_result := handle_delete_hosted_file(account_id=commons.x_account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan):
         return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
     elif hosted_file_delete_result is None:
@@ -524,116 +637,6 @@ async def delete_hosted_file(
     else:
         log.error(f'Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.')
         return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something may have gone wrong while trying to delete the hosted file from the server or the hosted_file record.') # Bad Request
-
-
-
-    # # ### SECTION ### Handle links NOTE NOTE NOTE NOTE NOTE NOTE
-    # # NOTE: If link_to_type and link_to_id passed then try and remove that link record first.
-
-    # # if hosted_file_result := delete_hosted_file_link(
-    # #         account_id = commons.x_account_id,
-    # #         hosted_file_id = hosted_file_id,
-
-    # #         link_to_type = link_to_type,
-    # #         link_to_id = link_to_id,
-
-    # #         rm_orphan = rm_orphan,
-    # #         ):
-    # #     pass
-    # # elif hosted_file_result is None:
-    # #     log.error('The hosted file link record was not found and may have already been deleted.')
-    # #     return mk_resp(data=False, status_code=404, response=commons.response) # Not Found
-    # # else:
-    # #     log.error('Something went wrong while trying to delete the hosted file link record.')
-    # #     return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
-
-    # # ### SECTION ### Handle orphan check and deletion of hosted_file record and file on server NOTE NOTE NOTE NOTE NOTE NOTE
-    # # NOTE: If not rm_orphan then do nothing else.
-    # # NOTE: If rm_orphan then get list of links for file.
-    # # NOTE:     If 0 links result then delete the hosted_file record and file on the server.
-    # # NOTE:     If >0 links result then do nothing else.
-
-    # # NOTE: Don't check or remove orphan
-    # if not rm_orphan:
-    #     log.info('Remove hosted file link. No orphan check.')
-    #     return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
-
-
-    # if hosted_file_delete_result := handle_delete_hosted_file(account_id=account_id, hosted_file_id=hosted_file_id, link_to_type=link_to_type, link_to_id=link_to_id, rm_orphan=rm_orphan):
-    #     return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
-    # else:
-    #     pass
-
-    # # # NOTE: Check and remove orphan
-    # # if hosted_file_link_rec_list_result := get_hosted_file_link_rec_list(hosted_file_id=hosted_file_id):
-    # #     hosted_file_link_result_list = []
-    # #     for hosted_file_link_rec in hosted_file_link_rec_list_result:
-    # #         hosted_file_link_result_list.append(hosted_file_link_rec)
-
-    # #     log.debug(hosted_file_link_result_list)
-    # #     hosted_file_list = hosted_file_link_result_list
-    # #     # NOT safe to delete the hosted_file record and file from server!!!
-    # #     # STOP!
-    # #     log.info('Remove hosted file link. Not an orphan file.')
-    # #     return mk_resp(data=True, response=commons.response, status_message='The hosted file link was deleted. Not an orphan file.')
-    # # elif isinstance(hosted_file_link_rec_list_result, list) or hosted_file_link_rec_list_result is None:
-    # #     hosted_file_list = []
-    # #     # Safe to delete the hosted_file record and file from server???
-    # #     # CONTINUE
-    # #     delete_hosted_file(hosted_file_id=hosted_file_id)
-    # # else:
-    # #     hosted_file_list = False
-    # #     # Safe to delete the hosted_file record and file from server???
-    # #     # CONTINUE
-
-
-    # # hosted_files_path = settings.FILES_PATH['hosted_files_root']
-    # hosted_files_path = '/home/scott/tmp/hosted_files_dev/'
-    # log.info(f'Hosted Files Path: {hosted_files_path}')
-
-    # if hosted_file_obj := load_hosted_file_obj(
-    #         hosted_file_id = hosted_file_id,
-    #         # inc_hosted_file = True,
-    #         inc_hosted_file_link_list = rm_orphan, # if rm_orphan then need to include hosted_file_link_list
-    #         ):
-    #     pass
-    # else:
-    #     return mk_resp(data=False, status_code=400, response=commons.response) # Bad Request
-
-    # # if not filename:
-    # #     filename = hosted_file_obj.filename
-    # # log.info(f'Filename: {filename}')
-    # dir_path = hosted_file_obj.directory_path
-    # subdir_path = hosted_file_obj.subdirectory_path
-    # hash_sha256 = hosted_file_obj.hash_sha256
-    # hash_filename = hash_sha256+'.file'
-
-    # if subdir_path:
-    #     full_subdirectory_path = os.path.join(hosted_files_path, subdir_path)
-    # else:
-    #     full_subdirectory_path = hosted_files_path
-    # log.debug(full_subdirectory_path)
-    # # pathlib.Path(full_subdirectory_path).mkdir(parents=True, exist_ok=True)
-    # file_path_w_subdir = os.path.join(full_subdirectory_path, hash_filename)
-    # log.info(f'Full file path with subdirectory: {file_path_w_subdir}')
-
-    # if os.path.exists(file_path_w_subdir):
-    #     log.info('File exists!')
-    #     if rm_orphan:
-    #         log.info('Going remove the file if it is an orphan...')
-    #         # try:
-    #         #     pathlib.Path(file_path_w_subdir).unlink()
-    #         # except OSError as e:
-    #         #     log.error("Error: %s : %s" % (file_path, e.strerror))
-    #         #     return mk_resp(data=False, status_code=400, response=commons.response, status_message='Something went wrong while trying to delete the hosted file from the server.') # Bad Request
-    #         return mk_resp(data=True, response=commons.response, status_message='The orphan hosted file was deleted from the server.')
-    #     else:
-    #         log.info('Remove hosted file link. No orphan check.')
-    #         return mk_resp(data=True, response=commons.response, status_message='The hosted file file link was deleted. No orphan check.')
-
-    # else:
-    #     log.error(f'The hosted file was not found on the server. Hash: {hash_sha256}')
-    #     return mk_resp(data=False, status_code=400, response=commons.response, status_message='The hosted file was not found on the server.') # Bad Request
 # ### END ### API Hosted File ### download_hosted_file() ###