Work on file uploads and listing event files.
This commit is contained in:
Scott Idem
@@ -74,6 +74,8 @@ def sql_insert(sql:str|None=None, data:dict|None=None, table_name:str|None=None,
|
||||
result_insert = db.execute(sql_insert, data)
|
||||
trans.commit()
|
||||
except Exception as e:
|
||||
# http://sqlalche.me/e/14/gkpj
|
||||
# Need a check for this: sqlalchemy.exc.IntegrityError: (MySQLdb._exceptions.IntegrityError) (1062, "Duplicate entry 'z-yyyy-xxxx-wwww for key 'PRIMARY'"
|
||||
trans.rollback()
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
@@ -514,7 +516,7 @@ def sql_delete(
|
||||
sql:str|None=None,
|
||||
data:dict|None=None
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if table_name and (record_id or record_id_random) and not (field_name or field_value or sql or data):
|
||||
@@ -735,7 +737,43 @@ def redis_lookup_id_random(record_id_random:int|str, table_name:str):
|
||||
log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.error('We should not be here. Something unexpected happened.')
|
||||
return False # Just in case
|
||||
# ### BEGIN ### API Lib General ### redis_lookup_id_random() ###
|
||||
# ### END ### API Lib General ### redis_lookup_id_random() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Lib General ### lookup_id_random() ###
|
||||
def lookup_id_random(record_id:int, table_name:str):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
data = { 'id': record_id }
|
||||
sql = f"""
|
||||
SELECT id_random
|
||||
FROM `{table_name}` AS `table`
|
||||
WHERE `table`.id = :id;
|
||||
"""
|
||||
|
||||
if select_results := sql_select(sql=sql, data=data):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(select_results)
|
||||
log.debug(type(select_results))
|
||||
if isinstance(select_results, dict):
|
||||
log.info(f"""Record ID found: {str(select_results['id_random'])}""")
|
||||
if record_id_random := select_results.get('id_random'):
|
||||
return str(record_id_random)
|
||||
else:
|
||||
log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.error('The SQL result was not what was expected.')
|
||||
return False
|
||||
else:
|
||||
log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.error('More than one record may have been found. There may be a duplicate id.')
|
||||
log.error(select_results)
|
||||
return False
|
||||
else:
|
||||
#log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.info('Record ID random was not found')
|
||||
return None
|
||||
# ### END ### API Lib General ### lookup_id_random() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Lib General ### lookup_id_random_pop() ###
|
||||
|
||||
18
app/main.py
18
app/main.py
@@ -18,7 +18,7 @@ from app.lib_general import *
|
||||
from app.log import log
|
||||
|
||||
# Import the routers here first:
|
||||
from app.routers import api_crud, api, account, address, archive, archive_content, contact, event, event_exhibit, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, lookup, membership, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
|
||||
from app.routers import api_crud, api, account, address, archive, archive_content, contact, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, lookup, membership, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
|
||||
|
||||
from app.db_sql import db
|
||||
|
||||
@@ -69,6 +69,14 @@ app.include_router(
|
||||
#dependencies=[Depends(get_account_header)],
|
||||
#responses={404: {'description': 'Not found'}},
|
||||
)
|
||||
# app.include_router(
|
||||
# flask_cfg.router,
|
||||
# prefix='/redis',
|
||||
# tags=['Redis'],
|
||||
# #dependencies=[Depends(get_token_header)],
|
||||
# #dependencies=[Depends(get_account_header)],
|
||||
# #responses={404: {'description': 'Not found'}},
|
||||
# )
|
||||
|
||||
app.include_router(
|
||||
account.router,
|
||||
@@ -126,6 +134,14 @@ app.include_router(
|
||||
#dependencies=[Depends(get_account_header)],
|
||||
#responses={404: {'description': 'Not found'}},
|
||||
)
|
||||
app.include_router(
|
||||
event_file.router,
|
||||
prefix='/event/file',
|
||||
tags=['Event File'],
|
||||
#dependencies=[Depends(get_token_header)],
|
||||
#dependencies=[Depends(get_account_header)],
|
||||
#responses={404: {'description': 'Not found'}},
|
||||
)
|
||||
app.include_router(
|
||||
event_person.router,
|
||||
prefix='/event/person',
|
||||
|
||||
159
app/methods/event_file_methods.py
Normal file
159
app/methods/event_file_methods.py
Normal file
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
import datetime, hashlib, os, pathlib, shutil, time
|
||||
|
||||
from fastapi import File, UploadFile
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
||||
from app.lib_general import log, logging
|
||||
|
||||
from app.models.event_file_models import Event_File_Base
|
||||
|
||||
|
||||
# ### BEGIN ### API Event File Methods ### create_event_file_obj() ###
|
||||
def create_event_file_obj(event_file_obj_new:Event_File_Base):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# event_file_obj_data = event_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'})
|
||||
event_file_obj_data = event_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'saved', 'already_exists', 'copy_timer', 'created_on', 'updated_on'})
|
||||
|
||||
if event_file_obj_in_result := sql_insert(data=event_file_obj_data, table_name='event_file', rm_id_random=True, id_random_length=8): pass
|
||||
else:
|
||||
return False
|
||||
|
||||
log.debug(event_file_obj_in_result)
|
||||
|
||||
event_file_id = event_file_obj_in_result
|
||||
|
||||
log.debug(f'Returning the new event_file_id: {event_file_id}')
|
||||
return event_file_id
|
||||
# ### END ### API Event File Methods ### create_event_file_obj() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Event File Methods ### load_event_file_obj() ###
|
||||
def load_event_file_obj(
|
||||
event_file_id: int|str,
|
||||
limit: int = 1000,
|
||||
model_as_dict: bool = False,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_hosted_file: bool = False,
|
||||
) -> Event_File_Base|dict|bool:
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if event_file_id := redis_lookup_id_random(record_id_random=event_file_id, table_name='event_file'): pass
|
||||
else: return False
|
||||
|
||||
# NOTE: What table or view should be used here???
|
||||
if event_file_rec := sql_select(table_name='v_event_file_simple', record_id=event_file_id):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(event_file_rec)
|
||||
else:
|
||||
return False
|
||||
|
||||
try:
|
||||
event_file_obj = Event_File_Base(**event_file_rec)
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(event_file_obj)
|
||||
except ValidationError as e:
|
||||
log.error(e.json())
|
||||
return False
|
||||
|
||||
# if inc_hosted_file:
|
||||
# x_id = event_file_rec.get('x_id', None)
|
||||
# if x_obj_result := load_x_obj(x_id=x_id):
|
||||
# x_obj = x_obj_result
|
||||
# event_file_obj.x = x_obj
|
||||
# else: event_file_obj.x = None
|
||||
|
||||
# model_as_dict = True
|
||||
|
||||
if model_as_dict:
|
||||
return event_file_obj.dict(by_alias=True, exclude_unset=False) # pylint: disable=no-member
|
||||
else:
|
||||
return event_file_obj
|
||||
# ### END ### API Event File Methods ### load_event_file_obj() ###
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Event File Methods ### load_event_file_obj_list() ###
|
||||
def load_event_file_obj_list(
|
||||
event_id: int|str|None = None,
|
||||
event_session_id: int|str|None = None,
|
||||
limit: int = 1000,
|
||||
model_as_dict: bool = False,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_hosted_file: bool = False,
|
||||
) -> list|bool:
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
data: dict = {}
|
||||
if event_id:
|
||||
if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass
|
||||
else: return False
|
||||
data['for_type'] = 'event'
|
||||
data['for_id'] = event_id
|
||||
sql_obj_type_id = f'`tbl`.for_type = :for_type AND `tbl`.for_id = :for_id'
|
||||
elif event_session_id:
|
||||
if event_session_id := redis_lookup_id_random(record_id_random=event_session_id, table_name='event_session'): pass
|
||||
else: return False
|
||||
data['for_type'] = 'event_session'
|
||||
data['for_id'] = event_session_id
|
||||
sql_obj_type_id = f'`tbl`.for_type = :for_type AND `tbl`.for_id = :for_id'
|
||||
|
||||
if enabled in ['enabled', 'disabled', 'all']:
|
||||
if enabled == 'enabled':
|
||||
data['enable'] = True
|
||||
sql_enabled = f'AND `tbl`.enable = :enable'
|
||||
elif enabled == 'disabled':
|
||||
data['enable'] = False
|
||||
sql_enabled = f'AND `tbl`.enable = :enable'
|
||||
elif enabled == 'all':
|
||||
sql_enabled = ''
|
||||
# else: tbl_obj['account'] = None
|
||||
|
||||
if limit:
|
||||
data['limit'] = limit
|
||||
sql_limit = f'LIMIT :limit'
|
||||
else:
|
||||
sql_limit = ''
|
||||
|
||||
sql = f"""
|
||||
SELECT `tbl`.id AS 'event_file_id', `tbl`.id_random AS 'event_file_id_random'
|
||||
FROM `event_file` AS `tbl`
|
||||
WHERE
|
||||
{sql_obj_type_id}
|
||||
{sql_enabled}
|
||||
ORDER BY `tbl`.created_on DESC, `tbl`.updated_on DESC
|
||||
{sql_limit};
|
||||
"""
|
||||
|
||||
if event_file_rec_li_result := sql_select(data=data, sql=sql, as_list=True):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(event_file_rec_li_result)
|
||||
event_file_result_li = []
|
||||
for event_file_rec in event_file_rec_li_result:
|
||||
event_file_id = event_file_rec.get('event_file_id', None)
|
||||
if event_file_result := load_event_file_obj(
|
||||
event_file_id = event_file_id,
|
||||
limit = limit,
|
||||
model_as_dict = True, # NOTE: This is overriding the model_as_dict param!
|
||||
enabled = enabled,
|
||||
inc_hosted_file = inc_hosted_file,
|
||||
):
|
||||
log.debug(event_file_result)
|
||||
event_file_result_li.append(event_file_result)
|
||||
else:
|
||||
log.debug(event_file_result)
|
||||
event_file_result_li.append(None)
|
||||
# log.debug(event_file_result_li)
|
||||
else:
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(event_file_rec_li_result)
|
||||
event_file_result_li = []
|
||||
|
||||
return event_file_result_li
|
||||
# ### END ### API Event Methods ### load_event_file_obj_list() ###
|
||||
@@ -8,7 +8,7 @@ from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_updat
|
||||
from app.lib_general import log, logging
|
||||
|
||||
# from app.methods.event_methods import load_event_obj
|
||||
# from app.methods.event_file_methods import load_event_file_obj
|
||||
from app.methods.event_file_methods import load_event_file_obj_list
|
||||
from app.methods.event_location_methods import load_event_location_obj
|
||||
from app.methods.event_person_methods import load_event_person_obj, update_event_person_obj
|
||||
from app.methods.event_presentation_methods import load_event_presentation_obj
|
||||
@@ -22,8 +22,9 @@ from app.models.event_session_models import Event_Session_Base
|
||||
# ### BEGIN ### API Event Session Methods ### load_event_session_obj() ###
|
||||
def load_event_session_obj(
|
||||
event_session_id: int|str,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
limit: int = 1000,
|
||||
model_as_dict: bool = False,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_address: bool = False,
|
||||
inc_contact: bool = False,
|
||||
inc_event_abstract_list: bool = False,
|
||||
@@ -74,6 +75,16 @@ def load_event_session_obj(
|
||||
if inc_event_device_list: pass
|
||||
if inc_event_file_list: pass
|
||||
|
||||
if inc_event_file_list:
|
||||
if event_file_dict_list := load_event_file_obj_list(
|
||||
event_session_id = event_session_id,
|
||||
limit = limit,
|
||||
model_as_dict = model_as_dict,
|
||||
enabled = enabled,
|
||||
):
|
||||
event_session_obj.event_file_list = event_file_dict_list
|
||||
else: event_session_obj.event_file_list = []
|
||||
|
||||
if inc_event_location and event_location_id:
|
||||
if event_location_obj := load_event_location_obj(
|
||||
event_location_id=event_location_id,
|
||||
@@ -181,7 +192,10 @@ def load_event_session_obj(
|
||||
event_session_obj.poc_event_person = poc_event_person_obj
|
||||
log.debug(event_session_obj)
|
||||
|
||||
return event_session_obj
|
||||
if model_as_dict:
|
||||
return event_session_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member
|
||||
else:
|
||||
return event_session_obj
|
||||
|
||||
|
||||
# ### BEGIN ### API Event Session Methods ### update_event_session_obj() ###
|
||||
|
||||
249
app/methods/hosted_file_methods.py
Normal file
249
app/methods/hosted_file_methods.py
Normal file
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
import datetime, hashlib, os, pathlib, shutil, time
|
||||
|
||||
from fastapi import File, UploadFile
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.db_sql import redis_lookup_id_random, sql_insert, sql_select, sql_update
|
||||
from app.lib_general import log, logging
|
||||
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ###
|
||||
def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'})
|
||||
hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'saved', 'already_exists', 'copy_timer', 'created_on', 'updated_on'})
|
||||
|
||||
if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass
|
||||
else:
|
||||
return False
|
||||
|
||||
log.debug(hosted_file_obj_in_result)
|
||||
|
||||
hosted_file_id = hosted_file_obj_in_result
|
||||
|
||||
log.debug(f'Returning the new hosted_file_id: {hosted_file_id}')
|
||||
return hosted_file_id
|
||||
# ### END ### API Hosted File Methods ### create_hosted_file_obj() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### load_hosted_file_obj() ###
|
||||
def load_hosted_file_obj(
|
||||
hosted_file_id: int|str,
|
||||
limit: int = 1000,
|
||||
model_as_dict: bool = False,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
# inc_x: bool = False,
|
||||
) -> Hosted_File_Base|dict|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return False
|
||||
|
||||
if hosted_file_rec := sql_select(table_name='v_hosted_file', record_id=hosted_file_id):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(hosted_file_rec)
|
||||
else:
|
||||
return False
|
||||
|
||||
try:
|
||||
hosted_file_obj = Hosted_File_Base(**hosted_file_rec)
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(hosted_file_obj)
|
||||
except ValidationError as e:
|
||||
log.error(e.json())
|
||||
return False
|
||||
|
||||
# if inc_x:
|
||||
# x_id = hosted_file_rec.get('x_id', None)
|
||||
# if x_obj_result := load_x_obj(x_id=x_id):
|
||||
# x_obj = x_obj_result
|
||||
# hosted_file_obj.x = x_obj
|
||||
# else: hosted_file_obj.x = None
|
||||
|
||||
if model_as_dict:
|
||||
return hosted_file_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member
|
||||
else:
|
||||
return hosted_file_obj
|
||||
# ### END ### API Hosted File Methods ### load_hosted_file_obj() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
|
||||
async def get_file_object_hash(file_object:File):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# 4096 bytes is the current block size on my workstation and Linode server
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
block_size = 131072
|
||||
hash_value = hashlib.sha256()
|
||||
|
||||
timer_start = time.process_time()
|
||||
for chunk in iter(lambda: file_object.read(block_size), b""):
|
||||
hash_value.update(chunk)
|
||||
file_hash = hash_value.hexdigest()
|
||||
file_object.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
|
||||
return file_hash
|
||||
# ### END ### API Hosted File Route ### get_file_object_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### guess_file_extension() ###
|
||||
def guess_file_extension(filename:str):
|
||||
return filename.rsplit('.', 1)[1].lower()
|
||||
# ### END ### API Hosted File Route ### guess_file_extension() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### allowed_file_extension() ###
|
||||
def allowed_file_extension(extension:str, extension_list:list):
|
||||
return extension.lower() in extension_list # app.config['ALLOWED_EXTENSIONS']
|
||||
# ### END ### API Hosted File Route ### allowed_file_extension() ###
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### save_file() ###
|
||||
async def save_file(
|
||||
file: UploadFile,
|
||||
account_id: int,
|
||||
account_id_random: str,
|
||||
for_object_type: str,
|
||||
for_object_id: int,
|
||||
for_object_id_random: str,
|
||||
check_allowed_extension: bool = False,
|
||||
):
|
||||
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
|
||||
log.debug(dir(file))
|
||||
log.debug(f'{file.filename}')
|
||||
|
||||
file_info: dict = {}
|
||||
file_info['saved'] = None
|
||||
file_info['for_object_type'] = for_object_type
|
||||
file_info['for_object_id'] = for_object_id
|
||||
file_info['for_object_id_random'] = for_object_id_random
|
||||
file_info['filename'] = file.filename
|
||||
file_info['extension'] = guess_file_extension(filename=file.filename)
|
||||
|
||||
if check_allowed_extension:
|
||||
if allowed_file_extension(extension=file_info['extension'], extension_list=['jpg','png','webp']):
|
||||
file_info['extension_allowed'] = True
|
||||
else:
|
||||
file_info['extension_allowed'] = False
|
||||
file_info['saved'] = False
|
||||
return file_info
|
||||
else:
|
||||
file_info['extension_allowed'] = None
|
||||
|
||||
# There is a difference between Content-Type and MIME type.
|
||||
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
|
||||
file_info['content_type'] = file.content_type # might also include charset or other parameters
|
||||
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
|
||||
|
||||
file.file.seek(0, os.SEEK_END)
|
||||
file_size = file.file.tell()
|
||||
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
log.debug(file_size)
|
||||
file_info['size'] = file_size
|
||||
|
||||
file_hash = await get_file_object_hash(file.file)
|
||||
log.debug(file_hash)
|
||||
file_info['hash_sha256'] = file_hash
|
||||
|
||||
# 16384 bytes is the default
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
buffer_size = 524288
|
||||
|
||||
#f_src = open(file_src, 'rb')
|
||||
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||
|
||||
#file_dest = f'{hosted_file_path}{file.filename}'
|
||||
file_dest = f'{hosted_file_path}{file_hash}.file'
|
||||
|
||||
existing_file_check = pathlib.Path(file_dest)
|
||||
|
||||
if existing_file_check.exists():
|
||||
file_info['already_exists'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
file_info['already_exists'] = False
|
||||
try:
|
||||
f_dest = open(file_dest, 'wb')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
log.exception('***')
|
||||
log.exception(str(e))
|
||||
log.exception('^^^ exception ^^^')
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Route ### save_file() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### hosted_file_link() ###
|
||||
def create_hosted_file_link(
|
||||
account_id: int|str,
|
||||
hosted_file_id: int|str,
|
||||
for_object_type: str,
|
||||
for_object_id: int|str,
|
||||
# for_object_id_random: str,
|
||||
):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||
else: return False
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return False
|
||||
if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass
|
||||
else: return False
|
||||
|
||||
|
||||
hosted_file_link_data: dict = {}
|
||||
hosted_file_link_data['account_id'] = account_id
|
||||
hosted_file_link_data['hosted_file_id'] = hosted_file_id
|
||||
hosted_file_link_data['object_type'] = for_object_type
|
||||
hosted_file_link_data['object_id'] = for_object_id
|
||||
|
||||
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
|
||||
if hosted_file_link_data_in_result := sql_insert(data=hosted_file_link_data, table_name='hosted_file_link', id_random_length=0): pass # This should be improved
|
||||
else:
|
||||
# This should be improved
|
||||
log.warning('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.')
|
||||
log.warning('Something may have gone wrong while trying to create the hosted_file_link record.')
|
||||
log.warning('The hosted_file_link was probably created fine though.')
|
||||
return False
|
||||
|
||||
log.debug(hosted_file_link_data_in_result)
|
||||
return True
|
||||
|
||||
# ### END ### API Hosted File Route ### hosted_file_link() ###
|
||||
@@ -46,7 +46,7 @@ def load_post_comment_obj(
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
inc_person: bool = False,
|
||||
inc_user: bool = False,
|
||||
) -> Post_Comment_Base|bool:
|
||||
) -> Post_Comment_Base|dict|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
|
||||
144
app/models/event_file_models.py
Normal file
144
app/models/event_file_models.py
Normal file
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
import datetime, hashlib, logging, os, pytz, redis, secrets
|
||||
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
|
||||
|
||||
from app.db_sql import redis_lookup_id_random, lookup_id_random
|
||||
from app.lib_general import log, logging
|
||||
|
||||
from app.models.common_field_schema import base_fields, default_num_bytes
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
|
||||
|
||||
class Event_File_Base(BaseModel):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
id_random: Optional[str] = Field(
|
||||
**base_fields['event_file_id_random'],
|
||||
alias='event_file_id_random',
|
||||
default_factory=lambda:secrets.token_urlsafe(default_num_bytes),
|
||||
)
|
||||
id: Optional[int] = Field(
|
||||
#alias='event_file_id'
|
||||
)
|
||||
|
||||
hosted_file_id_random: Optional[str]
|
||||
hosted_file_id: Optional[int]
|
||||
|
||||
for_type: Optional[str]
|
||||
for_id: Optional[int] # NOTE: This is reversed with for_id_random
|
||||
for_id_random: Optional[str] # NOTE: This is reversed with for_id
|
||||
|
||||
event_id_random: Optional[str]
|
||||
event_id: Optional[int]
|
||||
event_exhibit_id_random: Optional[str]
|
||||
event_exhibit_id: Optional[int]
|
||||
event_location_id_random: Optional[str]
|
||||
event_location_id: Optional[int]
|
||||
event_presentation_id_random: Optional[str]
|
||||
event_presentation_id: Optional[int]
|
||||
event_presenter_id_random: Optional[str]
|
||||
event_presenter_id: Optional[int]
|
||||
event_session_id_random: Optional[str]
|
||||
event_session_id: Optional[int]
|
||||
event_track_id_random: Optional[str]
|
||||
event_track_id: Optional[int]
|
||||
|
||||
filename: Optional[str]
|
||||
extension: Optional[str]
|
||||
title: Optional[str]
|
||||
description: Optional[str]
|
||||
|
||||
lu_file_purpose_id: Optional[str]
|
||||
file_purpose: Optional[str]
|
||||
|
||||
enable: Optional[bool]
|
||||
enable_from: Optional[datetime.datetime] = None
|
||||
enable_to: Optional[datetime.datetime] = None
|
||||
|
||||
hide: Optional[bool]
|
||||
|
||||
priority: Optional[bool]
|
||||
sort: Optional[int]
|
||||
group: Optional[str]
|
||||
|
||||
# notes: Optional[str]
|
||||
created_on: Optional[datetime.datetime] = None
|
||||
updated_on: Optional[datetime.datetime] = None
|
||||
|
||||
# Including other related objects
|
||||
hosted_file: Optional[Union[Hosted_File_Base, None]]
|
||||
|
||||
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
|
||||
|
||||
#@validator('event_file_id_random', always=True)
|
||||
def event_file_id_random_copy(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['id_random']:
|
||||
return values['id_random']
|
||||
return None
|
||||
|
||||
@validator('id', always=True)
|
||||
def event_file_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['id_random']:
|
||||
log.debug(values['id_random'])
|
||||
return redis_lookup_id_random(record_id_random=values['id_random'], table_name='event_file')
|
||||
return None
|
||||
|
||||
@validator('hosted_file_id', always=True)
|
||||
def hosted_file_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['hosted_file_id_random']:
|
||||
return redis_lookup_id_random(record_id_random=values['hosted_file_id_random'], table_name='hosted_file')
|
||||
return None
|
||||
|
||||
@validator('event_id', always=True)
|
||||
def event_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['event_id_random']:
|
||||
return redis_lookup_id_random(record_id_random=values['event_id_random'], table_name='event')
|
||||
return None
|
||||
|
||||
@validator('event_location_id', always=True)
|
||||
def event_location_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['event_location_id_random']:
|
||||
return redis_lookup_id_random(record_id_random=values['event_location_id_random'], table_name='event_location')
|
||||
return None
|
||||
|
||||
@validator('event_track_id', always=True)
|
||||
def event_track_id_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.WARNING)
|
||||
log.debug(locals())
|
||||
|
||||
if values['event_track_id_random']:
|
||||
return redis_lookup_id_random(record_id_random=values['event_track_id_random'], table_name='event_track')
|
||||
return None
|
||||
|
||||
class Config:
|
||||
underscore_attrs_are_private = True
|
||||
fields = base_fields
|
||||
|
||||
@validator('for_id_random', always=True)
|
||||
def for_id_random_lookup(cls, v, values, **kwargs):
|
||||
log.setLevel(logging.DEBUG)
|
||||
log.debug(locals())
|
||||
|
||||
if values['for_id'] and values['for_type']:
|
||||
return lookup_id_random(record_id=values['for_id'], table_name=values['for_type'])
|
||||
return None
|
||||
|
||||
#Event_File_Base.update_forward_refs()
|
||||
@@ -4,14 +4,14 @@ from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, stat
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
|
||||
from app.lib_general import *
|
||||
from app.lib_general import log, logging
|
||||
from app.config import settings
|
||||
from app.db_sql import *
|
||||
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
|
||||
|
||||
from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
|
||||
from app.models.api_models import Api_Base
|
||||
from app.models.response_models import *
|
||||
from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
@@ -167,3 +167,19 @@ async def delete_api_obj(
|
||||
obj_id=obj_id,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@router.get('/get_id/{object_type}/{object_id_random}', response_model=Resp_Body_Base)
|
||||
async def get_api_object_id(
|
||||
object_type: str = Query(..., min_length=1, max_length=50),
|
||||
object_id_random: str = Query(..., min_length=1, max_length=22),
|
||||
x_account_id: str = Header(...),
|
||||
by_alias: Optional[bool] = True,
|
||||
exclude_unset: Optional[bool] = True,
|
||||
):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if object_id := redis_lookup_id_random(record_id_random=object_id_random, table_name=object_type):
|
||||
return mk_resp(data={ 'object_id': object_id}, status_code=400)
|
||||
else: return mk_resp(data=None, status_code=400)
|
||||
|
||||
@@ -20,7 +20,7 @@ from app.models.event_models import *
|
||||
from app.models.event_abstract_models import *
|
||||
from app.models.event_badge_models import *
|
||||
from app.models.event_exhibit_models import *
|
||||
#from app.models.event_file_models import *
|
||||
from app.models.event_file_models import *
|
||||
from app.models.event_location_models import *
|
||||
from app.models.event_person_models import *
|
||||
from app.models.event_presentation_models import *
|
||||
@@ -28,6 +28,7 @@ from app.models.event_presenter_models import *
|
||||
from app.models.event_registration_models import *
|
||||
from app.models.event_session_models import *
|
||||
from app.models.event_track_models import *
|
||||
from app.models.hosted_file_models import *
|
||||
from app.models.journal_entry_models import *
|
||||
from app.models.membership_models import *
|
||||
from app.models.order_models import *
|
||||
@@ -66,7 +67,7 @@ obj_type_li['event_badge'] = {'table_name': 'event_badge', 'base_name': Event_Ba
|
||||
#obj_type_li['event_badge_template'] = {'table_name': 'event_badge_template', 'base_name': Event_Badge_Template_Base}
|
||||
#obj_type_li['event_device'] = {'table_name': 'event_device', 'base_name': Event_Device_Base}
|
||||
obj_type_li['event_exhibit'] = {'table_name': 'v_event_exhibit', 'base_name': Event_Exhibit_Base} # NOTE check view name: *_detail?
|
||||
#obj_type_li['event_file'] = {'table_name': 'v_event_file', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file
|
||||
obj_type_li['event_file'] = {'table_name': 'v_event_file', 'base_name': Event_File_Base} # Should this eventually be changed to event_hosted_file
|
||||
obj_type_li['event_location'] = {'table_name': 'v_event_location', 'base_name': Event_Location_Base}
|
||||
obj_type_li['event_person'] = {'table_name': 'v_event_person', 'base_name': Event_Person_Base}
|
||||
obj_type_li['event_presentation'] = {'table_name': 'v_event_presentation', 'base_name': Event_Presentation_Base}
|
||||
@@ -74,7 +75,7 @@ obj_type_li['event_presenter'] = {'table_name': 'v_event_presenter', 'base_name'
|
||||
obj_type_li['event_registration'] = {'table_name': 'v_event_registration', 'base_name': Event_Registration_Base}
|
||||
obj_type_li['event_session'] = {'table_name': 'v_event_session', 'base_name': Event_Session_Base}
|
||||
obj_type_li['event_track'] = {'table_name': 'v_event_track', 'base_name': Event_Track_Base}
|
||||
#obj_type_li['hosted_file'] = {'table_name': 'hosted_file', 'base_name': Hosted_File_Base}
|
||||
obj_type_li['hosted_file'] = {'table_name': 'v_hosted_file', 'base_name': Hosted_File_Base}
|
||||
#obj_type_li['hosted_file_link'] = {'table_name': 'hosted_file_link', 'base_name': Hosted_File_Link_Base}
|
||||
#obj_type_li['journal'] = {'table_name': 'v_journal', 'base_name': Journal_Base}
|
||||
obj_type_li['journal_entry'] = {'table_name': 'v_journal_entry', 'base_name': Journal_Entry_Base}
|
||||
|
||||
41
app/routers/event_file.py
Normal file
41
app/routers/event_file.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import datetime
|
||||
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, status
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
|
||||
from app.lib_general import log, logging
|
||||
from app.config import settings
|
||||
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
|
||||
|
||||
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
|
||||
from app.methods.event_file_methods import create_event_file_obj, load_event_file_obj # , update_event_file_obj
|
||||
|
||||
from app.models.event_file_models import Event_File_Base
|
||||
from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post('', response_model=Resp_Body_Base)
|
||||
async def post_event_file_obj(
|
||||
obj: Event_File_Base,
|
||||
x_account_id: str = Header(...),
|
||||
return_obj: Optional[bool] = True,
|
||||
by_alias: Optional[bool] = True,
|
||||
exclude_unset: Optional[bool] = True,
|
||||
):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
obj_type = 'event_file'
|
||||
obj_data_dict = obj.dict(by_alias=False, exclude_unset=True)
|
||||
result = post_obj_template(
|
||||
obj_type=obj_type,
|
||||
data=obj_data_dict,
|
||||
return_obj=True,
|
||||
by_alias=True,
|
||||
exclude_unset=True,
|
||||
)
|
||||
return result
|
||||
@@ -1,5 +1,5 @@
|
||||
from __future__ import annotations
|
||||
import datetime, hashlib, os, pathlib, shutil, time
|
||||
# import datetime, hashlib, os, pathlib, shutil, time
|
||||
#from datetime import datetime, time, timedelta
|
||||
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
@@ -11,6 +11,8 @@ from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select,
|
||||
|
||||
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
|
||||
from app.methods.hosted_file_methods import create_hosted_file_obj, load_hosted_file_obj, save_file, create_hosted_file_link
|
||||
|
||||
from app.models.hosted_file_models import Hosted_File_Base
|
||||
from app.models.response_models import mk_resp
|
||||
|
||||
@@ -22,7 +24,7 @@ router = APIRouter()
|
||||
# This just needs to return the currect model for a hosted_file
|
||||
# Everything else seems to be working well
|
||||
# Should this also do something with meta data and updating the DB?
|
||||
@router.post('/upload_files/')
|
||||
@router.post('/upload_files')
|
||||
async def upload_files(
|
||||
file_list: List[UploadFile] = File(...),
|
||||
account_id: str = Form(..., min_length=1, max_length=22),
|
||||
@@ -30,6 +32,7 @@ async def upload_files(
|
||||
for_object_type: str = Form(...),
|
||||
for_object_id: str = Form(..., min_length=1, max_length=22),
|
||||
check_allowed_extension: bool = False,
|
||||
# create_hosted_file_link: bool = True,
|
||||
x_account_id: str = Header(..., ),
|
||||
return_obj: bool = True,
|
||||
by_alias: bool = True,
|
||||
@@ -127,237 +130,175 @@ async def upload_files(
|
||||
hosted_file_dict['copy_timer'] = file_info['copy_timer']
|
||||
hosted_file_list.append(hosted_file_dict)
|
||||
|
||||
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
|
||||
if create_hosted_file_link(
|
||||
account_id=account_id,
|
||||
hosted_file_id=hosted_file_id,
|
||||
for_object_type=for_object_type,
|
||||
for_object_id=for_object_id,
|
||||
): pass # This if statement should be improved
|
||||
else:
|
||||
# This if statement should be improved
|
||||
log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.')
|
||||
log.debug('Something may have gone wrong while trying to create the hosted_file_link record.')
|
||||
log.debug('The hosted_file_link was probably created fine though.')
|
||||
|
||||
log.debug(hosted_file_list)
|
||||
return mk_resp(data=hosted_file_list)
|
||||
# ### END ### API Hosted File Route ### upload_files() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### save_file() ###
|
||||
async def save_file(
|
||||
file: UploadFile,
|
||||
account_id: int,
|
||||
account_id_random: str,
|
||||
# ### BEGIN ### API Hosted File Route ### upload_files_fake() ###
|
||||
# This just needs to return the currect model for a hosted_file
|
||||
# Everything else seems to be working well
|
||||
# Should this also do something with meta data and updating the DB?
|
||||
@router.post('/upload_files/fake')
|
||||
async def upload_files_fake(
|
||||
file_info_li: list,
|
||||
account_id: str,
|
||||
# filename: Optional[str] = Form(...),
|
||||
for_object_type: str,
|
||||
for_object_id: int,
|
||||
for_object_id_random: str,
|
||||
for_object_id: str,
|
||||
check_allowed_extension: bool = False,
|
||||
|
||||
# create_hosted_file_link: bool = True,
|
||||
x_account_id: str = Header(..., ),
|
||||
return_obj: bool = True,
|
||||
by_alias: bool = True,
|
||||
exclude_unset: bool = True,
|
||||
):
|
||||
# log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
|
||||
log.debug(file_info_li)
|
||||
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
account_id_random = account_id # This is for the account random str ID
|
||||
if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
|
||||
else:
|
||||
return mk_resp(data=None, status_code=400)
|
||||
|
||||
log.debug(dir(file))
|
||||
log.debug(f'{file.filename}')
|
||||
for_object_type = for_object_type
|
||||
for_object_id_random = for_object_id # This is for the object random str ID
|
||||
if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass
|
||||
else:
|
||||
return mk_resp(data=None, status_code=400)
|
||||
|
||||
file_info = {}
|
||||
file_info['saved'] = None
|
||||
file_info['for_object_type'] = for_object_type
|
||||
file_info['for_object_id'] = for_object_id
|
||||
file_info['for_object_id_random'] = for_object_id_random
|
||||
file_info['filename'] = file.filename
|
||||
file_info['extension'] = guess_file_extension(filename=file.filename)
|
||||
|
||||
if check_allowed_extension:
|
||||
if allowed_file_extension:
|
||||
file_info['extension_allowed'] = True
|
||||
hosted_file_list = []
|
||||
for file_info in file_info_li:
|
||||
if file_info['saved']:
|
||||
# Create a new host_file object entry
|
||||
log.info('Check and create a new host_file object entry...')
|
||||
if file_info['already_exists']:
|
||||
# Look up in DB based on hash
|
||||
# Get existing host_file object_entry and existing host_file.id_random.
|
||||
log.info('Look up in DB based on hash...')
|
||||
if hosted_file_sel_result := sql_select(
|
||||
table_name = 'hosted_file',
|
||||
field_name = 'hash_sha256',
|
||||
field_value = file_info['hash_sha256'],
|
||||
):
|
||||
hosted_file_id = hosted_file_sel_result.get('id_random', None)
|
||||
# hosted_file_obj = Hosted_File_Base(**file_info)
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(hosted_file_dict)
|
||||
else:
|
||||
# SOMETHING WENT WRONG
|
||||
# Going to try and create a new host_file entry...
|
||||
log.warning('For some reason a host_file object entry with the has was not found.')
|
||||
# file_info['id_random'] = None
|
||||
hosted_file_obj = Hosted_File_Base(**file_info)
|
||||
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
|
||||
hosted_file_id = hosted_file_obj_result
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
else:
|
||||
log.warning('For some reason a host_file object entry could not be created.')
|
||||
hosted_file_id = None
|
||||
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
|
||||
log.debug(hosted_file_obj_result)
|
||||
log.debug(hosted_file_sel_result)
|
||||
else:
|
||||
# Just in case look up in DB based on hash
|
||||
log.info('Look up in DB based on hash...')
|
||||
if hosted_file_sel_result := sql_select(
|
||||
table_name = 'hosted_file',
|
||||
field_name = 'hash_sha256',
|
||||
field_value = file_info['hash_sha256'],
|
||||
):
|
||||
log.warning('Found an existing host_file object_entry in the DB but the file was not found on the server!')
|
||||
# Got existing host_file object_entry!
|
||||
# Odd... the hash was found in the database, but the file had to be copied again.
|
||||
# If this happens then the file on the host server was probably deleted at some point.
|
||||
hosted_file_id = hosted_file_sel_result.get('id_random', None)
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
else:
|
||||
# This is normal since the file was not found on the host server and not found in the DB.
|
||||
# Create a new host_file object entry and new host_file.id_random.
|
||||
log.warning('This is sort of normal. The file may have been deleted from the host server...')
|
||||
hosted_file_obj = Hosted_File_Base(**file_info)
|
||||
if hosted_file_obj_result := create_hosted_file_obj(hosted_file_obj_new=hosted_file_obj):
|
||||
hosted_file_id = hosted_file_obj_result
|
||||
hosted_file_dict = load_hosted_file_obj(hosted_file_id=hosted_file_id, model_as_dict=True)
|
||||
else:
|
||||
log.warning('For some reason a host_file object entry could not be created.')
|
||||
hosted_file_id = None
|
||||
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
|
||||
log.debug(hosted_file_obj_result)
|
||||
log.debug(hosted_file_sel_result)
|
||||
else:
|
||||
file_info['extension_allowed'] = False
|
||||
file_info['saved'] = False
|
||||
return file_info
|
||||
else:
|
||||
file_info['extension_allowed'] = None
|
||||
file_info['id_random'] = None
|
||||
hosted_file_obj = Hosted_File_Base(**file_info)
|
||||
hosted_file_id = None
|
||||
hosted_file_dict = hosted_file_obj.dict(by_alias=True, exclude_unset=True, exclude={'id', 'id_random'}) # pylint: disable=no-member
|
||||
|
||||
# There is a difference between Content-Type and MIME type.
|
||||
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
|
||||
file_info['content_type'] = file.content_type # might also include charset or other parameters
|
||||
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
|
||||
# file_info_obj = Hosted_File_Base(**file_info)
|
||||
hosted_file_dict['extension_allowed'] = file_info['extension_allowed']
|
||||
hosted_file_dict['already_exists'] = file_info['already_exists']
|
||||
hosted_file_dict['saved'] = file_info['saved']
|
||||
hosted_file_dict['copy_timer'] = file_info['copy_timer']
|
||||
log.debug(hosted_file_dict)
|
||||
hosted_file_list.append(hosted_file_dict)
|
||||
|
||||
file.file.seek(0, os.SEEK_END)
|
||||
file_size = file.file.tell()
|
||||
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
log.debug(file_size)
|
||||
file_info['size'] = file_size
|
||||
# NOTE: Currently sql_insert does not handel all successful inserts correctly. If there is not an autonum ID then it will return 0 as the ID.
|
||||
if create_hosted_file_link(
|
||||
account_id=account_id,
|
||||
hosted_file_id=hosted_file_id,
|
||||
for_object_type=for_object_type,
|
||||
for_object_id=for_object_id,
|
||||
): pass # This if statement should be improved
|
||||
else:
|
||||
# This if statement should be improved
|
||||
log.debug('Because the hosted_file_link table does not have a primary autonum this check is incorrect even when successful.')
|
||||
log.debug('Something may have gone wrong while trying to create the hosted_file_link record.')
|
||||
log.debug('The hosted_file_link was probably created fine though.')
|
||||
|
||||
file_hash = await get_file_object_hash(file.file)
|
||||
log.debug(file_hash)
|
||||
file_info['hash_sha256'] = file_hash
|
||||
|
||||
# 16384 bytes is the default
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
buffer_size = 524288
|
||||
|
||||
#f_src = open(file_src, 'rb')
|
||||
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
|
||||
|
||||
#file_dest = f'{hosted_file_path}{file.filename}'
|
||||
file_dest = f'{hosted_file_path}{file_hash}.file'
|
||||
|
||||
existing_file_check = pathlib.Path(file_dest)
|
||||
|
||||
if existing_file_check.exists():
|
||||
file_info['already_exists'] = True
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = True
|
||||
else:
|
||||
file_info['already_exists'] = False
|
||||
try:
|
||||
f_dest = open(file_dest, 'wb')
|
||||
timer_start = time.process_time()
|
||||
shutil.copyfileobj(f_src, f_dest, buffer_size)
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
file_info['copy_timer'] = elapsed_time
|
||||
file_info['saved'] = True
|
||||
except Exception as e:
|
||||
log.exception('*** An exception happened. ***')
|
||||
log.exception(repr(e))
|
||||
log.exception('***')
|
||||
log.exception(str(e))
|
||||
log.exception('^^^ exception ^^^')
|
||||
|
||||
file_info['copy_timer'] = 0
|
||||
file_info['saved'] = False
|
||||
log.debug(shutil.disk_usage(hosted_file_path))
|
||||
|
||||
return file_info
|
||||
# ### END ### API Hosted File Route ### save_file() ###
|
||||
log.debug(hosted_file_list)
|
||||
return mk_resp(data=hosted_file_list)
|
||||
# ### END ### API Hosted File Route ### upload_files_fake() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### get_file_object_hash() ###
|
||||
async def get_file_object_hash(file_object:File):
|
||||
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
# 4096 bytes is the current block size on my workstation and Linode server
|
||||
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
|
||||
block_size = 131072
|
||||
hash_value = hashlib.sha256()
|
||||
|
||||
timer_start = time.process_time()
|
||||
for chunk in iter(lambda: file_object.read(block_size), b""):
|
||||
hash_value.update(chunk)
|
||||
file_hash = hash_value.hexdigest()
|
||||
file_object.seek(0) # The file will not properly save if seek is not reset to 0.
|
||||
timer_end = time.process_time()
|
||||
elapsed_time = timer_end - timer_start
|
||||
log.debug(f'Elapsed time: {elapsed_time}')
|
||||
|
||||
return file_hash
|
||||
# ### END ### API Hosted File Route ### get_file_object_hash() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### guess_file_extension() ###
|
||||
def guess_file_extension(filename:str):
|
||||
return filename.rsplit('.', 1)[1].lower()
|
||||
# ### END ### API Hosted File Route ### guess_file_extension() ###
|
||||
|
||||
@router.post('/test_uploads')
|
||||
async def test_upload_files(
|
||||
file_list: List[UploadFile],
|
||||
# account_id: str = Form(..., min_length=1, max_length=22),
|
||||
# filename: Optional[str] = Form(...),
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### allowed_file_extension() ###
|
||||
def allowed_file_extension(extension:str):
|
||||
return extension.lower() in app.config['ALLOWED_EXTENSIONS']
|
||||
# ### END ### API Hosted File Route ### allowed_file_extension() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Route ### save_file() ###
|
||||
async def hosted_file_link(
|
||||
account_id: int,
|
||||
hosted_file_id: str,
|
||||
for_object_type: str,
|
||||
for_object_id: int,
|
||||
for_object_id_random: str,
|
||||
):
|
||||
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_file_link_data = {}
|
||||
hosted_file_link_data['account_id'] = account_id
|
||||
hosted_file_link_data['hosted_file_id'] = hosted_file_id
|
||||
hosted_file_link_data['object_type'] = for_object_type
|
||||
hosted_file_link_data['object_id'] = for_object_id
|
||||
for file_obj in file_list:
|
||||
file_info = await save_file(
|
||||
file = file_obj,
|
||||
account_id = account_id,
|
||||
account_id_random = account_id_random,
|
||||
for_object_type = for_object_type,
|
||||
for_object_id = for_object_id,
|
||||
for_object_id_random = for_object_id_random,
|
||||
check_allowed_extension = check_allowed_extension,
|
||||
)
|
||||
log.debug(file_info)
|
||||
|
||||
if response['data']['id'] == True:
|
||||
#print('Tried to insert a new hosted_file record, but there is a duplicate.')
|
||||
# There was likely a record with the same hash value.
|
||||
table_name = 'hosted_file'
|
||||
field_name = 'hash_sha256'
|
||||
field_value = data['hash_sha256']
|
||||
select_hosted_file_response = select_record(table_name=table_name, field_name=field_name, field_value=field_value)
|
||||
if select_hosted_file_response:
|
||||
hosted_file_link_data['hosted_file_id'] = select_hosted_file_response['id']
|
||||
response['data']['id'] = select_hosted_file_response['id']
|
||||
response['data']['id_random'] = select_hosted_file_response['id_random']
|
||||
else:
|
||||
return False
|
||||
else:
|
||||
print('Inserted new host_file record.')
|
||||
pass
|
||||
|
||||
table_name = 'hosted_file_link'
|
||||
|
||||
hosted_file_link_response = sql_insert_for_rest(data=hosted_file_link_data, table_name=table_name, sql=None, model=None, resource_ref=True)
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### create_hosted_file_obj() ###
|
||||
def create_hosted_file_obj(hosted_file_obj_new:Hosted_File_Base):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
hosted_file_obj_data = hosted_file_obj_new.dict(by_alias=False, exclude_defaults=False, exclude_unset=True, exclude={'created_on', 'updated_on'})
|
||||
|
||||
if hosted_file_obj_in_result := sql_insert(data=hosted_file_obj_data, table_name='hosted_file', rm_id_random=True, id_random_length=8): pass
|
||||
else:
|
||||
return False
|
||||
|
||||
log.debug(hosted_file_obj_in_result)
|
||||
|
||||
hosted_file_id = hosted_file_obj_in_result
|
||||
|
||||
log.debug(f'Returning the new hosted_file_id: {hosted_file_id}')
|
||||
return hosted_file_id
|
||||
# ### END ### API Hosted File Methods ### create_hosted_file_obj() ###
|
||||
|
||||
|
||||
# ### BEGIN ### API Hosted File Methods ### load_hosted_file_obj() ###
|
||||
def load_hosted_file_obj(
|
||||
hosted_file_id: int|str,
|
||||
limit: int = 1000,
|
||||
model_as_dict: bool = False,
|
||||
enabled: str = 'enabled', # enabled, disabled, all
|
||||
# inc_x: bool = False,
|
||||
) -> Hosted_File_Base|bool:
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if hosted_file_id := redis_lookup_id_random(record_id_random=hosted_file_id, table_name='hosted_file'): pass
|
||||
else: return False
|
||||
|
||||
if hosted_file_rec := sql_select(table_name='hosted_file', record_id=hosted_file_id):
|
||||
#log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(hosted_file_rec)
|
||||
else:
|
||||
return False
|
||||
|
||||
try:
|
||||
hosted_file_obj = Hosted_File_Base(**hosted_file_rec)
|
||||
log.debug(hosted_file_obj)
|
||||
except ValidationError as e:
|
||||
log.error(e.json())
|
||||
return False
|
||||
|
||||
# if inc_x:
|
||||
# x_id = hosted_file_rec.get('x_id', None)
|
||||
# if x_obj_result := load_x_obj(x_id=x_id):
|
||||
# x_obj = x_obj_result
|
||||
# hosted_file_obj.x = x_obj
|
||||
# else: hosted_file_obj.x = None
|
||||
|
||||
if model_as_dict:
|
||||
return hosted_file_obj.dict(by_alias=True, exclude_unset=True) # pylint: disable=no-member
|
||||
else:
|
||||
return hosted_file_obj
|
||||
# ### END ### API Hosted File Methods ### load_hosted_file_obj() ###
|
||||
return mk_resp(data=False, status_code=501)
|
||||
0
app/routers/hosted_file_help.py
Normal file
0
app/routers/hosted_file_help.py
Normal file
Reference in New Issue
Block a user