From 415e452988387744c075fce5d48891b535504c36 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Mon, 14 Jun 2021 17:57:15 -0400 Subject: [PATCH] Work on file uploads. More error checking. --- app/models/hosted_file_models.py | 4 + app/routers/hosted_file.py | 126 +++++++++++++++++++++++-------- 2 files changed, 97 insertions(+), 33 deletions(-) diff --git a/app/models/hosted_file_models.py b/app/models/hosted_file_models.py index 0e4393a..0c6c05c 100644 --- a/app/models/hosted_file_models.py +++ b/app/models/hosted_file_models.py @@ -44,6 +44,10 @@ class Hosted_File_Base(BaseModel): package_name: Optional[str] + already_exists: Optional[str] # This will probably only be populated on upload results + copy_timer: Optional[str] # This will probably only be populated on upload results + saved: Optional[str] # This will probably only be populated on upload results + #metadata: Optional[str] #hide: Optional[int] diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index 19ea409..c0fe5c8 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -49,7 +49,7 @@ async def upload_files( else: return mk_resp(data=None, status_code=400) - file_info_list = [] + hosted_file_list = [] for file_obj in file_list: file_info = await save_file( file = file_obj, @@ -68,55 +68,67 @@ async def upload_files( if hosted_file_sel_result := sql_select( table_name = 'hosted_file', field_name = 'hash_sha256', - filed_value = file_info['hash_sha256'], + field_value = file_info['hash_sha256'], ): - file_info['id_random'] = hosted_file_result.get('id_random', None) - hosted_file_obj = Hosted_File_Base(**file_info) + hosted_file_id = hosted_file_sel_result.get('id_random', None) + # hosted_file_obj = Hosted_File_Base(**file_info) + hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: # SOMETHING WENT WRONG # Going to try and create a new host_file entry... log.warning('For some reason a host_file object entry with the has was not found.') # file_info['id_random'] = None - hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) - if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): - hosted_file.id_random = hosted_file_result.get('id_random', None) + hosted_file_obj = Hosted_File_Base(**file_info) + if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): + hosted_file_id = hosted_file_obj_result + hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: log.warning('For some reason a host_file object entry could not be created.') - hosted_file.id_random = None - log.debug(hosted_file_in_result) + hosted_file_id = None + hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member + log.debug(hosted_file_obj_result) log.debug(hosted_file_sel_result) else: # Just in case look up in DB based on hash if hosted_file_sel_result := sql_select( table_name = 'hosted_file', field_name = 'hash_sha256', - filed_value = file_info['hash_sha256'], + field_value = file_info['hash_sha256'], ): log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') # Got existing host_file object_entry! # Odd... the hash was found in the database, but the file had to be copied again. # If this happens then the file on the host server was probably deleted at some point. - file_info['id_random'] = hosted_file_result.get('id_random', None) - hosted_file_obj = Hosted_File_Base(**file_info) + hosted_file_id = hosted_file_sel_result.get('id_random', None) + hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: # This is normal since the file was not found on the host server and not found in the DB. # Create a new host_file object entry and new host_file.id_random. - hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) - if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): - file_info['id_random'] = hosted_file_result.get('id_random', None) + hosted_file_obj = Hosted_File_Base(**file_info) + if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj): + hosted_file_id = hosted_file_obj_result + hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True) else: log.warning('For some reason a host_file object entry could not be created.') - file_info['id_random'] = None - log.debug(hosted_file_in_result) + hosted_file_id = None + hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member + log.debug(hosted_file_obj_result) log.debug(hosted_file_sel_result) else: file_info['id_random'] = None + hosted_file_obj = Hosted_File_Base(**file_info) + hosted_file_id = None + hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member # file_info_obj = Hosted_File_Base(**file_info) - file_info_list.append(file_info_obj) + hosted_file_dict['extension_allowed'] = file_info['extension_allowed'] + hosted_file_dict['already_exists'] = file_info['already_exists'] + hosted_file_dict['saved'] = file_info['saved'] + hosted_file_dict['copy_timer'] = file_info['copy_timer'] + hosted_file_list.append(hosted_file_dict) - log.debug(file_info_list) - return mk_resp(data=file_info_list) + log.debug(hosted_file_list) + return mk_resp(data=hosted_file_list) # ### END ### API Hosted File Route ### upload_files() ### @@ -189,19 +201,27 @@ async def save_file( if existing_file_check.exists(): file_info['already_exists'] = True file_info['copy_timer'] = 0 + file_info['saved'] = True else: file_info['already_exists'] = False - f_dest = open(file_dest, 'wb') - - timer_start = time.process_time() - shutil.copyfileobj(f_src, f_dest, buffer_size) - timer_end = time.process_time() - elapsed_time = timer_end - timer_start - log.debug(f'Elapsed time: {elapsed_time}') - file_info['copy_timer'] = elapsed_time - - file_info['saved'] = True + try: + f_dest = open(file_dest, 'wb') + timer_start = time.process_time() + shutil.copyfileobj(f_src, f_dest, buffer_size) + timer_end = time.process_time() + elapsed_time = timer_end - timer_start + log.debug(f'Elapsed time: {elapsed_time}') + file_info['copy_timer'] = elapsed_time + file_info['saved'] = True + except Exception as e: + log.exception('*** An exception happened. ***') + log.exception(repr(e)) + log.exception('***') + log.exception(str(e)) + log.exception('^^^ exception ^^^') + file_info['copy_timer'] = 0 + file_info['saved'] = False log.debug(shutil.disk_usage(hosted_file_path)) return file_info @@ -282,13 +302,12 @@ async def hosted_file_link( hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True) - # ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ### def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base): log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'user', 'created_on', 'updated_on'}) + hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'}) if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass else: @@ -300,4 +319,45 @@ def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base): log.debug(f'Returning the new hosted_file_id: {hosted_file_id}') return hosted_file_id -# ### END ### API Hosted File Methods ### create_hosted_file_obj() ### \ No newline at end of file +# ### END ### API Hosted File Methods ### create_hosted_file_obj() ### + + +# ### BEGIN ### API Hosted File Methods ### load_hosted_file_obj() ### +def load_hosted_file_obj( + hosted_file_id: int|str, + limit: int = 1000, + model_as_dict: bool = False, + enabled: str = 'enabled', # enabled, disabled, all + # inc_x: bool = False, + ) -> Hosted_File_Base|bool: + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass + else: return False + + if hosted_file_rec := sql_select(table_name='hosted_file', record_id=hosted_file_id): + #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(hosted_file_rec) + else: + return False + + try: + hosted_file_obj = Hosted_File_Base(**hosted_file_rec) + log.debug(hosted_file_obj) + except ValidationError as e: + log.error(e.json()) + return False + + # if inc_x: + # x_id = hosted_file_rec.get('x_id', None) + # if x_obj_result := load_x_obj(x_id=x_id): + # x_obj = x_obj_result + # hosted_file_obj.x = x_obj + # else: hosted_file_obj.x = None + + if model_as_dict: + return hosted_file_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member + else: + return hosted_file_obj +# ### END ### API Hosted File Methods ### load_hosted_file_obj() ### \ No newline at end of file