Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unplanned work for file uploads

Browse Source
This commit is contained in:
Scott Idem
2021-06-14 16:04:37 -04:00
parent fe232b8cba
commit eb23d16ad5
7 changed files with 164 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/routers/account.py
View File

@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
@@ -13,7 +13,7 @@ from app.methods.account_methods import load_account_obj
from app.methods.account_cfg_methods import load_account_cfg_obj
from app.models.account_models import Account_Base
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()

2
app/routers/api_crud.py
View File

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import *
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random, lookup_id_random_pop
from app.models.response_models import *

4
app/routers/event.py
View File

@@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import *
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.event_methods import load_event_obj, update_event_obj
from app.models.event_models import Event_Base
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()

4
app/routers/event_person.py
View File

@@ -5,7 +5,7 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
@@ -16,7 +16,7 @@ from app.methods.user_methods import create_user_obj, load_user_obj, update_user
from app.models.event_person_models import Event_Person_New_Base, Event_Person_Base
from app.models.person_models import Person_Base
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
from app.models.user_models import User_New_Base, User_Base

213
app/routers/hosted_file.py
View File

@@ -1,87 +1,84 @@
import datetime, shutil, time
from __future__ import annotations
import datetime, hashlib, os, pathlib, shutil, time
#from datetime import datetime, time, timedelta
from fastapi import APIRouter, Body, Depends, File, Form, Header, HTTPException, Query, Response, status, UploadFile
from pydantic import BaseModel, EmailStr, Field
from typing import Dict, List, Optional, Set, Union
from app.lib_general import *
from ..log import *
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
# from .api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.models.hosted_file_models import Hosted_File_Base
from app.models.response_models import *
from app.models.response_models import mk_resp
router = APIRouter()
# This just needs to return the currect model for a hosted_file
# Everything else seems to be workign well
# Everything else seems to be working well
# Should this also do something with meta data and updating the DB?
@router.post('/upload_files/')
async def create_upload_files(
files: List[UploadFile] = File(...),
file_list: List[UploadFile] = File(...),
account_id: str = Form(..., min_length=1, max_length=22),
filename: Optional[str] = Form(...),
object_type: Optional[str] = Form(...),
object_id: Optional[str] = Form(..., min_length=1, max_length=22),
# filename: Optional[str] = Form(...),
for_object_type: str = Form(...),
for_object_id: str = Form(..., min_length=1, max_length=22),
x_account_id: Optional[str] = Header(..., ),
return_obj: Optional[bool] = True,
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
):
data = {}
data['account_id'] = account_id
#data['filename'] = filename
data['object_type'] = object_type
data['object_id'] = object_id
data['file_li'] = []
account_id_random = account_id # This is for the account random str ID
if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
else:
return mk_resp(data=None, status_code=404)
log.debug(await save_file_li(files))
for_object_type = for_object_type
for_object_id_random = for_object_id # This is for the object random str ID
if for_object_id := redis_lookup_id_random(record_id_random=for_object_id, table_name=for_object_type): pass
else:
return mk_resp(data=None, status_code=404)
response_li = []
for file in files:
file_info = {}
file_info['filename'] = file.filename
file_info['content_type'] = file.content_type
# file_info_li = await save_file_li(file_list, for_object_type, for_object_id, for_object_id_random)
file.file.seek(0, os.SEEK_END)
file_size = file.file.tell()
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
file_info['size'] = file_size
log.debug(file_size)
file_info_list = []
for file_obj in file_list:
file_info = await save_file(file=file_obj, account_id=account_id, account_id_random=account_id_random, for_object_type=for_object_type, for_object_id=for_object_id, for_object_id_random=for_object_id_random)
file_info_list.append(file_info)
data['file_li'].append(file_info)
return mk_resp(data=data)
#response = { 'response_data': response_data, 'response_li':response_li }
#response['filenames'] = [file.filename for file in files]
#return response
#return {'filenames': [file.filename for file in files]}
log.debug(file_info_list)
return mk_resp(data=file_info_list)
# This is not needed. Just for reference of the mk_resp() function
@router.post('/', response_model=Resp_Body_Base)
async def post_upload_file(
account_id: str = Query(..., min_length=1, max_length=22),
# @router.post('/', response_model=Resp_Body_Base)
# async def post_upload_file(
# account_id: str = Query(..., min_length=1, max_length=22),
# ):
# if return_obj:
# if order_cart_obj := load_order_cart_obj(order_cart_id=order_cart_id, inc_order_cart_line_li=inc_order_cart_line_li, inc_order_cart_cfg=inc_order_cart_cfg):
# data = order_cart_obj.dict(by_alias=True, exclude_unset=False)
# return mk_resp(data=data)
# else:
# return mk_resp(data=False, status_code=404) # Not Found
# else:
# return mk_resp(data=True)
async def save_file_li(
file_li: List[UploadFile], # = File(...),
account_id: int,
account_id_random: str,
for_object_type: str,
for_object_id: int,
for_object_id_random: str,
):
if return_obj:
if order_cart_obj := load_order_cart_obj(order_cart_id=order_cart_id, inc_order_cart_line_li=inc_order_cart_line_li, inc_order_cart_cfg=inc_order_cart_cfg):
data = order_cart_obj.dict(by_alias=True, exclude_unset=False)
return mk_resp(data=data)
else:
return mk_resp(data=False, status_code=404) # Not Found
else:
return mk_resp(data=True)
async def save_file_li(file_li: List[UploadFile] = File(...)):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -89,18 +86,32 @@ async def save_file_li(file_li: List[UploadFile] = File(...)):
log.debug(shutil.disk_usage(hosted_file_path))
result = []
for file in file_li:
file_info_li = []
for file in file_li.g:
log.debug(file)
log.debug(f'{file.filename}')
file_info = {}
file_info['for_object_type'] = for_object_type
file_info['for_object_id'] = for_object_id
file_info['for_object_id_random'] = for_object_id_random
file_info['filename'] = file.filename
# There is a difference between Content-Type and MIME type.
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
file_info['content_type'] = file.content_type # might also include charset or other parameters
file_info['mimetype'] = file.mimetype
file.file.seek(0, os.SEEK_END)
file_size = file.file.tell()
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
#file_info['size'] = request_file_size
log.debug(file_size)
file_info['size'] = file_size
file_hash = await get_file_object_hash(file.file)
log.debug(file_hash)
file_info['hash_sha256'] = file_hash
# 16384 bytes is the default
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
@@ -112,6 +123,89 @@ async def save_file_li(file_li: List[UploadFile] = File(...)):
#file_dest = f'{hosted_file_path}{file.filename}'
file_dest = f'{hosted_file_path}{file_hash}.file'
existing_file_check = pathlib.Path(file_dest)
if existing_file_check.exists():
file_info['already_exists'] = True
file_info['copy_timer'] = 0
else:
file_info['already_exists'] = False
f_dest = open(file_dest, 'wb')
timer_start = time.process_time()
shutil.copyfileobj(f_src, f_dest, buffer_size)
timer_end = time.process_time()
elapsed_time = timer_end - timer_start
log.debug(f'Elapsed time: {elapsed_time}')
file_info['copy_timer'] = elapsed_time
# result.append({ 'filename': file.filename })
file_info_li.append(file_info)
log.debug(shutil.disk_usage(hosted_file_path))
return file_info_li
async def save_file(
file: UploadFile,
account_id: int,
account_id_random: str,
for_object_type: str,
for_object_id: int,
for_object_id_random: str,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
hosted_file_path = '/home/scott/tmp/hosted_file_dev/'
log.debug(shutil.disk_usage(hosted_file_path))
log.debug(dir(file))
log.debug(f'{file.filename}')
file_info = {}
file_info['for_object_type'] = for_object_type
file_info['for_object_id'] = for_object_id
file_info['for_object_id_random'] = for_object_id_random
file_info['filename'] = file.filename
file_info['extension'] = guess_file_extension(filename=file.filename)
# There is a difference between Content-Type and MIME type.
# https://stackoverflow.com/questions/3452381/whats-the-difference-of-contenttype-and-mimetype
file_info['content_type'] = file.content_type # might also include charset or other parameters
# file_info['mimetype'] = file.mimetype # This may need to be filled in a different way?
file.file.seek(0, os.SEEK_END)
file_size = file.file.tell()
file.file.seek(0) # The file will not properly save if seek is not reset to 0.
#file_info['size'] = request_file_size
log.debug(file_size)
file_info['size'] = file_size
file_hash = await get_file_object_hash(file.file)
log.debug(file_hash)
file_info['hash_sha256'] = file_hash
# 16384 bytes is the default
# 4096 8192 16384 32768 65536 131072 262144 524288 1048576 bytes
buffer_size = 524288
#file_src = file.filename
#f_src = open(file_src, 'rb')
f_src = file.file # Don't need to do open(file_src, 'rb') since it is already "open"
#file_dest = f'{hosted_file_path}{file.filename}'
file_dest = f'{hosted_file_path}{file_hash}.file'
existing_file_check = pathlib.Path(file_dest)
if existing_file_check.exists():
file_info['already_exists'] = True
file_info['copy_timer'] = 0
else:
file_info['already_exists'] = False
f_dest = open(file_dest, 'wb')
timer_start = time.process_time()
@@ -119,15 +213,14 @@ async def save_file_li(file_li: List[UploadFile] = File(...)):
timer_end = time.process_time()
elapsed_time = timer_end - timer_start
log.debug(f'Elapsed time: {elapsed_time}')
result.append({ 'filename': file.filename })
file_info['copy_timer'] = elapsed_time
log.debug(shutil.disk_usage(hosted_file_path))
return result
return file_info
async def get_file_object_hash(file_object):
async def get_file_object_hash(file_object:File):
#log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
@@ -143,7 +236,7 @@ async def get_file_object_hash(file_object):
file_object.seek(0) # The file will not properly save if seek is not reset to 0.
timer_end = time.process_time()
elapsed_time = timer_end - timer_start
#log.debug(f'Elapsed time: {elapsed_time}')
log.debug(f'Elapsed time: {elapsed_time}')
return file_hash
@@ -153,8 +246,8 @@ async def get_file_object_hash(file_object):
# return '.' in filename and filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS']
# def guess_file_extension(filename):
# return filename.rsplit('.', 1)[1].lower()
def guess_file_extension(filename:str):
return filename.rsplit('.', 1)[1].lower()
# def copyLargeFile(src, dest, buffer_size=16000):
# with open(src, 'rb') as fsrc:

4
app/routers/person.py
View File

@@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.person_methods import load_person_obj, update_person_obj
from app.models.person_models import Person_Base
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()

4
app/routers/user.py
View File

@@ -5,14 +5,14 @@ from typing import Dict, List, Optional, Set, Union
from app.lib_general import log, logging, secure_hash_string, verify_secure_hash_string
from app.config import settings
from app.db_sql import *
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
from app.methods.user_methods import load_user_obj
from app.models.common_field_schema import default_num_bytes
from app.models.response_models import *
from app.models.response_models import Resp_Body_Base, mk_resp
from app.models.user_models import User_Base, User_New_Base, User_Out_Base