Work on file uploads and listing event files.

This commit is contained in:
Scott Idem
2021-06-15 18:05:56 -04:00
parent 415e452988
commit 0dc50e4509
12 changed files with 841 additions and 222 deletions

View File

@@ -4,14 +4,14 @@ from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, stat
from pydantic import BaseModel, EmailStr, Field
from typing import Dict, List, Optional, Set, Union
from app.lib_general import *
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.models.api_models import Api_Base
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()
@@ -167,3 +167,19 @@ async def delete_api_obj(
obj_id=obj_id,
)
return result
@router.get('/get_id/{object_type}/{object_id_random}', response_model=Resp_Body_Base)
async def get_api_object_id(
object_type: str = Query(..., min_length=1, max_length=50),
object_id_random: str = Query(..., min_length=1, max_length=22),
x_account_id: str = Header(...),
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if object_id := redis_lookup_id_random(record_id_random=object_id_random, table_name=object_type):
return mk_resp(data={ 'object_id': object_id}, status_code=400)
else: return mk_resp(data=None, status_code=400)

View File

@@ -20,7 +20,7 @@ from app.models.event_models import *
from app.models.event_abstract_models import *
from app.models.event_badge_models import *
from app.models.event_exhibit_models import *
#from app.models.event_file_models import *
from app.models.event_file_models import *
from app.models.event_location_models import *
from app.models.event_person_models import *
from app.models.event_presentation_models import *
@@ -28,6 +28,7 @@ from app.models.event_presenter_models import *
from app.models.event_registration_models import *
from app.models.event_session_models import *
from app.models.event_track_models import *
from app.models.hosted_file_models import *
from app.models.journal_entry_models import *
from app.models.membership_models import *
from app.models.order_models import *
@@ -66,7 +67,7 @@ obj_type_li['event_badge'] = {'table_name': 'event_badge', 'base_name': Event_Ba
#obj_type_li['event_badge_template'] = {'table_name': 'event_badge_template', 'base_name': Event_Badge_Template_Base}
#obj_type_li['event_device'] = {'table_name': 'event_device', 'base_name': Event_Device_Base}
obj_type_li['event_exhibit'] = {'table_name': 'v_event_exhibit', 'base_name': Event_Exhibit_Base} # NOTE check view name: *_detail?
#obj_type_li['event_file'] = {'table_name': 'v_event_file', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file
obj_type_li['event_file'] = {'table_name': 'v_event_file', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file
obj_type_li['event_location'] = {'table_name': 'v_event_location', 'base_name': Event_Location_Base}
obj_type_li['event_person'] = {'table_name': 'v_event_person', 'base_name': Event_Person_Base}
obj_type_li['event_presentation'] = {'table_name': 'v_event_presentation', 'base_name': Event_Presentation_Base}
@@ -74,7 +75,7 @@ obj_type_li['event_presenter'] = {'table_name': 'v_event_presenter', 'base_name'
obj_type_li['event_registration'] = {'table_name': 'v_event_registration', 'base_name': Event_Registration_Base}
obj_type_li['event_session'] = {'table_name': 'v_event_session', 'base_name': Event_Session_Base}
obj_type_li['event_track'] = {'table_name': 'v_event_track', 'base_name': Event_Track_Base}
#obj_type_li['hosted_file'] = {'table_name': 'hosted_file', 'base_name': Hosted_File_Base}
obj_type_li['hosted_file'] = {'table_name': 'v_hosted_file', 'base_name': Hosted_File_Base}
#obj_type_li['hosted_file_link'] = {'table_name': 'hosted_file_link', 'base_name': Hosted_File_Link_Base}
#obj_type_li['journal'] = {'table_name': 'v_journal', 'base_name': Journal_Base}
obj_type_li['journal_entry'] = {'table_name': 'v_journal_entry', 'base_name': Journal_Entry_Base}

41
app/routers/event_file.py Normal file
View File

@@ -0,0 +1,41 @@
import datetime
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, status
from pydantic import BaseModel, EmailStr, Field
from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.event_file_methods import create_event_file_obj, load_event_file_obj # , update_event_file_obj
from app.models.event_file_models import Event_File_Base
from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()
@router.post('', response_model=Resp_Body_Base)
async def post_event_file_obj(
obj: Event_File_Base,
x_account_id: str = Header(...),
return_obj: Optional[bool] = True,
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
obj_type = 'event_file'
obj_data_dict = obj.dict(by_alias=False, exclude_unset=True)
result = post_obj_template(
obj_type=obj_type,
data=obj_data_dict,
return_obj=True,
by_alias=True,
exclude_unset=True,
)
return result

View File

@@ -1,5 +1,5 @@
from __future__ import annotations
import datetime, hashlib, os, pathlib, shutil, time
# import datetime, hashlib, os, pathlib, shutil, time
#from datetime import datetime, time, timedelta
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
from pydantic import BaseModel, EmailStr, Field
@@ -11,6 +11,8 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link
from app.models.hosted_file_models import Hosted_File_Base
from app.models.response_models import mk_resp
@@ -22,7 +24,7 @@ router = APIRouter()
# This just needs to return the currect model for a hosted_file
# Everything else seems to be working well
# Should this also do something with meta data and updating the DB?
@router.post('/upload_files/')
@router.post('/upload_files')
async def upload_files(
file_list: List[UploadFile] = File(...),
account_id: str = Form(..., min_length=1, max_length=22),
@@ -30,6 +32,7 @@ async def upload_files(
for_object_type: str = Form(...),
for_object_id: str = Form(..., min_length=1, max_length=22),
check_allowed_extension: bool = False,
# create_hosted_file_link: bool = True,
x_account_id: str = Header(..., ),
return_obj: bool = True,
by_alias: bool = True,
@@ -127,237 +130,175 @@ async def upload_files(
hosted_file_dict['copy_timer'] = file_info['copy_timer']
hosted_file_list.append(hosted_file_dict)
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
if create_hosted_file_link(
account_id=account_id,
hosted_file_id=hosted_file_id,
for_object_type=for_object_type,
for_object_id=for_object_id,
): pass # This if statement should be improved
else:
# This if statement should be improved
log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.')
log.debug('Something may have gone wrong while trying to create the hosted_file_link record.')
log.debug('The hosted_file_link was probably created fine though.')
log.debug(hosted_file_list)
return mk_resp(data=hosted_file_list)
# ### END ### API Hosted File Route ### upload_files() ###
# ### BEGIN ### API Hosted File Route ### save_file() ###
async def save_file(
file: UploadFile,
account_id: int,
account_id_random: str,
# ### BEGIN ### API Hosted File Route ### upload_files_fake() ###
# This just needs to return the currect model for a hosted_file
# Everything else seems to be working well
# Should this also do something with meta data and updating the DB?
@router.post('/upload_files/fake')
async def upload_files_fake(
file_info_li: list,
account_id: str,
# filename: Optional[str] = Form(...),
for_object_type: str,
for_object_id: int,
for_object_id_random: str,
for_object_id: str,
check_allowed_extension: bool = False,
# create_hosted_file_link: bool = True,
x_account_id: str = Header(..., ),
return_obj: bool = True,
by_alias: bool = True,
exclude_unset: bool = True,
):
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
log.debug(file_info_li)
log.debug(shutil.disk_usage(hosted_file_path))
account_id_random = account_id # This is for the account random str ID
if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
else:
return mk_resp(data=None, status_code=400)
log.debug(dir(file))
log.debug(f'{file.filename}')
for_object_type = for_object_type
for_object_id_random = for_object_id # This is for the object random str ID
if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass
else:
return mk_resp(data=None, status_code=400)
file_info = {}
file_info['saved'] = None
file_info['for_object_type'] = for_object_type
file_info['for_object_id'] = for_object_id
file_info['for_object_id_random'] = for_object_id_random
file_info['filename'] = file.filename
file_info['extension'] = guess_file_extension(filename=file.filename)
if check_allowed_extension:
if allowed_file_extension:
file_info['extension_allowed'] = True
hosted_file_list = []
for file_info in file_info_li:
if file_info['saved']:
# Create a new host_file object entry
log.info('Check and create a new host_file object entry...')
if file_info['already_exists']:
# Look up in DB based on hash
# Get existing host_file object_entry and existing host_file.id_random.
log.info('Look up in DB based on hash...')
if hosted_file_sel_result := sql_select(
table_name = 'hosted_file',
field_name = 'hash_sha256',
field_value = file_info['hash_sha256'],
):
hosted_file_id = hosted_file_sel_result.get('id_random', None)
# hosted_file_obj = Hosted_File_Base(**file_info)
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(hosted_file_dict)
else:
# SOMETHING WENT WRONG
# Going to try and create a new host_file entry...
log.warning('For some reason a host_file object entry with the has was not found.')
# file_info['id_random'] = None
hosted_file_obj = Hosted_File_Base(**file_info)
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
hosted_file_id = hosted_file_obj_result
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else:
log.warning('For some reason a host_file object entry could not be created.')
hosted_file_id = None
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
log.debug(hosted_file_obj_result)
log.debug(hosted_file_sel_result)
else:
# Just in case look up in DB based on hash
log.info('Look up in DB based on hash...')
if hosted_file_sel_result := sql_select(
table_name = 'hosted_file',
field_name = 'hash_sha256',
field_value = file_info['hash_sha256'],
):
log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!')
# Got existing host_file object_entry!
# Odd... the hash was found in the database, but the file had to be copied again.
# If this happens then the file on the host server was probably deleted at some point.
hosted_file_id = hosted_file_sel_result.get('id_random', None)
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else:
# This is normal since the file was not found on the host server and not found in the DB.
# Create a new host_file object entry and new host_file.id_random.
log.warning('This is sort of normal. The file may have been deleted from the host server...')
hosted_file_obj = Hosted_File_Base(**file_info)
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
hosted_file_id = hosted_file_obj_result
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
else:
log.warning('For some reason a host_file object entry could not be created.')
hosted_file_id = None
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
log.debug(hosted_file_obj_result)
log.debug(hosted_file_sel_result)
else:
file_info['extension_allowed'] = False
file_info['saved'] = False
return file_info
else:
file_info['extension_allowed'] = None
file_info['id_random'] = None
hosted_file_obj = Hosted_File_Base(**file_info)
hosted_file_id = None
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
# There is a difference between Content-Type and MIME type.
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
file_info['content_type'] = file.content_type # might also include charset or other parameters
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
# file_info_obj = Hosted_File_Base(**file_info)
hosted_file_dict['extension_allowed'] = file_info['extension_allowed']
hosted_file_dict['already_exists'] = file_info['already_exists']
hosted_file_dict['saved'] = file_info['saved']
hosted_file_dict['copy_timer'] = file_info['copy_timer']
log.debug(hosted_file_dict)
hosted_file_list.append(hosted_file_dict)
file.file.seek(0, os.SEEK_END)
file_size = file.file.tell()
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
log.debug(file_size)
file_info['size'] = file_size
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
if create_hosted_file_link(
account_id=account_id,
hosted_file_id=hosted_file_id,
for_object_type=for_object_type,
for_object_id=for_object_id,
): pass # This if statement should be improved
else:
# This if statement should be improved
log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.')
log.debug('Something may have gone wrong while trying to create the hosted_file_link record.')
log.debug('The hosted_file_link was probably created fine though.')
file_hash = await get_file_object_hash(file.file)
log.debug(file_hash)
file_info['hash_sha256'] = file_hash
# 16384 bytes is the default
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
buffer_size = 524288
#f_src = open(file_src, 'rb')
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
#file_dest = f'{hosted_file_path}{file.filename}'
file_dest = f'{hosted_file_path}{file_hash}.file'
existing_file_check = pathlib.Path(file_dest)
if existing_file_check.exists():
file_info['already_exists'] = True
file_info['copy_timer'] = 0
file_info['saved'] = True
else:
file_info['already_exists'] = False
try:
f_dest = open(file_dest, 'wb')
timer_start = time.process_time()
shutil.copyfileobj(f_src, f_dest, buffer_size)
timer_end = time.process_time()
elapsed_time = timer_end - timer_start
log.debug(f'Elapsed time: {elapsed_time}')
file_info['copy_timer'] = elapsed_time
file_info['saved'] = True
except Exception as e:
log.exception('*** An exception happened. ***')
log.exception(repr(e))
log.exception('***')
log.exception(str(e))
log.exception('^^^ exception ^^^')
file_info['copy_timer'] = 0
file_info['saved'] = False
log.debug(shutil.disk_usage(hosted_file_path))
return file_info
# ### END ### API Hosted File Route ### save_file() ###
log.debug(hosted_file_list)
return mk_resp(data=hosted_file_list)
# ### END ### API Hosted File Route ### upload_files_fake() ###
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
async def get_file_object_hash(file_object:File):
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# 4096 bytes is the current block size on my workstation and Linode server
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
block_size = 131072
hash_value = hashlib.sha256()
timer_start = time.process_time()
for chunk in iter(lambda: file_object.read(block_size), b""):
hash_value.update(chunk)
file_hash = hash_value.hexdigest()
file_object.seek(0) # The file will not properly save if seek is not reset to 0.
timer_end = time.process_time()
elapsed_time = timer_end - timer_start
log.debug(f'Elapsed time: {elapsed_time}')
return file_hash
# ### END ### API Hosted File Route ### get_file_object_hash() ###
# ### BEGIN ### API Hosted File Route ### guess_file_extension() ###
def guess_file_extension(filename:str):
return filename.rsplit('.', 1)[1].lower()
# ### END ### API Hosted File Route ### guess_file_extension() ###
@router.post('/test_uploads')
async def test_upload_files(
file_list: List[UploadFile],
# account_id: str = Form(..., min_length=1, max_length=22),
# filename: Optional[str] = Form(...),
# ### BEGIN ### API Hosted File Route ### allowed_file_extension() ###
def allowed_file_extension(extension:str):
return extension.lower() in app.config['ALLOWED_EXTENSIONS']
# ### END ### API Hosted File Route ### allowed_file_extension() ###
# ### BEGIN ### API Hosted File Route ### save_file() ###
async def hosted_file_link(
account_id: int,
hosted_file_id: str,
for_object_type: str,
for_object_id: int,
for_object_id_random: str,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
hosted_file_link_data = {}
hosted_file_link_data['account_id'] = account_id
hosted_file_link_data['hosted_file_id'] = hosted_file_id
hosted_file_link_data['object_type'] = for_object_type
hosted_file_link_data['object_id'] = for_object_id
for file_obj in file_list:
file_info = await save_file(
file = file_obj,
account_id = account_id,
account_id_random = account_id_random,
for_object_type = for_object_type,
for_object_id = for_object_id,
for_object_id_random = for_object_id_random,
check_allowed_extension = check_allowed_extension,
)
log.debug(file_info)
if response['data']['id'] == True:
#print('Tried to insert a new hosted_file record, but there is a duplicate.')
# There was likely a record with the same hash value.
table_name = 'hosted_file'
field_name = 'hash_sha256'
field_value = data['hash_sha256']
select_hosted_file_response = select_record(table_name=table_name, field_name=field_name, field_value=field_value)
if select_hosted_file_response:
hosted_file_link_data['hosted_file_id'] = select_hosted_file_response['id']
response['data']['id'] = select_hosted_file_response['id']
response['data']['id_random'] = select_hosted_file_response['id_random']
else:
return False
else:
print('Inserted new host_file record.')
pass
table_name = 'hosted_file_link'
hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True)
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ###
def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base):
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'})
if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass
else:
return False
log.debug(hosted_file_obj_in_result)
hosted_file_id = hosted_file_obj_in_result
log.debug(f'Returning the new hosted_file_id: {hosted_file_id}')
return hosted_file_id
# ### END ### API Hosted File Methods ### create_hosted_file_obj() ###
# ### BEGIN ### API Hosted File Methods ### load_hosted_file_obj() ###
def load_hosted_file_obj(
hosted_file_id: int|str,
limit: int = 1000,
model_as_dict: bool = False,
enabled: str = 'enabled', # enabled, disabled, all
# inc_x: bool = False,
) -> Hosted_File_Base|bool:
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
else: return False
if hosted_file_rec := sql_select(table_name='hosted_file', record_id=hosted_file_id):
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(hosted_file_rec)
else:
return False
try:
hosted_file_obj = Hosted_File_Base(**hosted_file_rec)
log.debug(hosted_file_obj)
except ValidationError as e:
log.error(e.json())
return False
# if inc_x:
# x_id = hosted_file_rec.get('x_id', None)
# if x_obj_result := load_x_obj(x_id=x_id):
# x_obj = x_obj_result
# hosted_file_obj.x = x_obj
# else: hosted_file_obj.x = None
if model_as_dict:
return hosted_file_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member
else:
return hosted_file_obj
# ### END ### API Hosted File Methods ### load_hosted_file_obj() ###
return mk_resp(data=False, status_code=501)

View File