Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Work on file uploads. More error checking.

Browse Source
This commit is contained in:
Scott Idem
2021-06-14 17:57:15 -04:00
parent 2d180006b6
commit 415e452988
2 changed files with 97 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/models/hosted_file_models.py
View File

@@ -44,6 +44,10 @@ class Hosted_File_Base(BaseModel):
package_name: Optional[str] package_name: Optional[str]
already_exists: Optional[str] # This will probably only be populated on upload results
copy_timer: Optional[str] # This will probably only be populated on upload results
saved: Optional[str] # This will probably only be populated on upload results
#metadata: Optional[str] #metadata: Optional[str]
#hide: Optional[int] #hide: Optional[int]

126
app/routers/hosted_file.py
View File

@@ -49,7 +49,7 @@ async def upload_files(
else: else:
return mk_resp(data=None, status_code=400) return mk_resp(data=None, status_code=400)
file_info_list = [] hosted_file_list = []
for file_obj in file_list: for file_obj in file_list:
file_info = await save_file( file_info = await save_file(
file = file_obj, file = file_obj,
@@ -68,55 +68,67 @@ async def upload_files(
if hosted_file_sel_result := sql_select( if hosted_file_sel_result := sql_select(
table_name = 'hosted_file', table_name = 'hosted_file',
field_name = 'hash_sha256', field_name = 'hash_sha256',
filed_value = file_info['hash_sha256'], field_value = file_info['hash_sha256'],
): ):
file_info['id_random'] = hosted_file_result.get('id_random', None) hosted_file_id = hosted_file_sel_result.get('id_random', None)
hosted_file_obj = Hosted_File_Base(**file_info) # hosted_file_obj = Hosted_File_Base(**file_info)
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else: else:
# SOMETHING WENT WRONG # SOMETHING WENT WRONG
# Going to try and create a new host_file entry... # Going to try and create a new host_file entry...
log.warning('For some reason a host_file object entry with the has was not found.') log.warning('For some reason a host_file object entry with the has was not found.')
# file_info['id_random'] = None # file_info['id_random'] = None
hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) hosted_file_obj = Hosted_File_Base(**file_info)
if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
hosted_file.id_random = hosted_file_result.get('id_random', None) hosted_file_id = hosted_file_obj_result
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else: else:
log.warning('For some reason a host_file object entry could not be created.') log.warning('For some reason a host_file object entry could not be created.')
hosted_file.id_random = None hosted_file_id = None
log.debug(hosted_file_in_result) hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
log.debug(hosted_file_obj_result)
log.debug(hosted_file_sel_result) log.debug(hosted_file_sel_result)
else: else:
# Just in case look up in DB based on hash # Just in case look up in DB based on hash
if hosted_file_sel_result := sql_select( if hosted_file_sel_result := sql_select(
table_name = 'hosted_file', table_name = 'hosted_file',
field_name = 'hash_sha256', field_name = 'hash_sha256',
filed_value = file_info['hash_sha256'], field_value = file_info['hash_sha256'],
): ):
log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!') log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!')
# Got existing host_file object_entry! # Got existing host_file object_entry!
# Odd... the hash was found in the database, but the file had to be copied again. # Odd... the hash was found in the database, but the file had to be copied again.
# If this happens then the file on the host server was probably deleted at some point. # If this happens then the file on the host server was probably deleted at some point.
file_info['id_random'] = hosted_file_result.get('id_random', None) hosted_file_id = hosted_file_sel_result.get('id_random', None)
hosted_file_obj = Hosted_File_Base(**file_info) hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else: else:
# This is normal since the file was not found on the host server and not found in the DB. # This is normal since the file was not found on the host server and not found in the DB.
# Create a new host_file object entry and new host_file.id_random. # Create a new host_file object entry and new host_file.id_random.
hosted_file_dict = Hosted_File_Base(**file_info).dict(by_alias=False, exclude_defaults=False, exclude_unset=True) hosted_file_obj = Hosted_File_Base(**file_info)
if hosted_file_in_result := sql_insert(data=hosted_file_dict, table_name='hosted_file'): if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
file_info['id_random'] = hosted_file_result.get('id_random', None) hosted_file_id = hosted_file_obj_result
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else: else:
log.warning('For some reason a host_file object entry could not be created.') log.warning('For some reason a host_file object entry could not be created.')
file_info['id_random'] = None hosted_file_id = None
log.debug(hosted_file_in_result) hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
log.debug(hosted_file_obj_result)
log.debug(hosted_file_sel_result) log.debug(hosted_file_sel_result)
else: else:
file_info['id_random'] = None file_info['id_random'] = None
hosted_file_obj = Hosted_File_Base(**file_info)
hosted_file_id = None
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
# file_info_obj = Hosted_File_Base(**file_info) # file_info_obj = Hosted_File_Base(**file_info)
file_info_list.append(file_info_obj) hosted_file_dict['extension_allowed'] = file_info['extension_allowed']
hosted_file_dict['already_exists'] = file_info['already_exists']
hosted_file_dict['saved'] = file_info['saved']
hosted_file_dict['copy_timer'] = file_info['copy_timer']
hosted_file_list.append(hosted_file_dict)
log.debug(file_info_list) log.debug(hosted_file_list)
return mk_resp(data=file_info_list) return mk_resp(data=hosted_file_list)
# ### END ### API Hosted File Route ### upload_files() ### # ### END ### API Hosted File Route ### upload_files() ###
@@ -189,19 +201,27 @@ async def save_file(
if existing_file_check.exists(): if existing_file_check.exists():
file_info['already_exists'] = True file_info['already_exists'] = True
file_info['copy_timer'] = 0 file_info['copy_timer'] = 0
file_info['saved'] = True
else: else:
file_info['already_exists'] = False file_info['already_exists'] = False
f_dest = open(file_dest, 'wb') try:
f_dest = open(file_dest, 'wb')
timer_start = time.process_time() timer_start = time.process_time()
shutil.copyfileobj(f_src, f_dest, buffer_size) shutil.copyfileobj(f_src, f_dest, buffer_size)
timer_end = time.process_time() timer_end = time.process_time()
elapsed_time = timer_end - timer_start elapsed_time = timer_end - timer_start
log.debug(f'Elapsed time: {elapsed_time}') log.debug(f'Elapsed time: {elapsed_time}')
file_info['copy_timer'] = elapsed_time file_info['copy_timer'] = elapsed_time
file_info['saved'] = True
file_info['saved'] = True except Exception as e:
log.exception('*** An exception happened. ***')
log.exception(repr(e))
log.exception('***')
log.exception(str(e))
log.exception('^^^ exception ^^^')
file_info['copy_timer'] = 0
file_info['saved'] = False
log.debug(shutil.disk_usage(hosted_file_path)) log.debug(shutil.disk_usage(hosted_file_path))
return file_info return file_info
@@ -282,13 +302,12 @@ async def hosted_file_link(
hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True) hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True)
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ### # ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ###
def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base): def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals()) log.debug(locals())
hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'user', 'created_on', 'updated_on'}) hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'})
if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass
else: else:
@@ -300,4 +319,45 @@ def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base):
log.debug(f'Returning the new hosted_file_id: {hosted_file_id}') log.debug(f'Returning the new hosted_file_id: {hosted_file_id}')
return hosted_file_id return hosted_file_id
# ### END ### API Hosted File Methods ### create_hosted_file_obj() ### # ### END ### API Hosted File Methods ### create_hosted_file_obj() ###
# ### BEGIN ### API Hosted File Methods ### load_hosted_file_obj() ###
def load_hosted_file_obj(
hosted_file_id: int|str,
limit: int = 1000,
model_as_dict: bool = False,
enabled: str = 'enabled', # enabled, disabled, all
# inc_x: bool = False,
) -> Hosted_File_Base|bool:
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
else: return False
if hosted_file_rec := sql_select(table_name='hosted_file', record_id=hosted_file_id):
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(hosted_file_rec)
else:
return False
try:
hosted_file_obj = Hosted_File_Base(**hosted_file_rec)
log.debug(hosted_file_obj)
except ValidationError as e:
log.error(e.json())
return False
# if inc_x:
# x_id = hosted_file_rec.get('x_id', None)
# if x_obj_result := load_x_obj(x_id=x_id):
# x_obj = x_obj_result
# hosted_file_obj.x = x_obj
# else: hosted_file_obj.x = None
if model_as_dict:
return hosted_file_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member
else:
return hosted_file_obj
# ### END ### API Hosted File Methods ### load_hosted_file_obj() ###