From d3f26f1696ba4517fde27ac1ab26cdfadd1f137f Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Fri, 26 Apr 2024 13:49:48 -0400 Subject: [PATCH] Upgrading to fastapi .95.1 --- admin/requirements.txt | 38 +++++++---- app/db_sql.py | 5 ++ app/routers/api_crud.py | 10 +-- development.env | 138 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 development.env diff --git a/admin/requirements.txt b/admin/requirements.txt index 8904714..fc3f6d5 100644 --- a/admin/requirements.txt +++ b/admin/requirements.txt @@ -1,20 +1,29 @@ +# Updated manually 2024-04-26 with a lot of trial and error. +# A few are commented out even though they are actually used and required. Other packages already pull them in. +# SQLAlchemy needs to be upgraded to 2.x. There are issues with async IO or something related to that. +# https://docs.sqlalchemy.org/en/14/changelog/migration_20.html + # aioredis # BAD! Not maintained! -anyio +aiofiles +# aiohttp # added 2024-04-24 +# anyio argon2-cffi argon2-cffi-bindings -asgiref +# asgiref async-timeout -certifi -cffi +baize # added 2023-08-17 +# certifi +# cffi charset-normalizer click Deprecated dnspython email-validator et-xmlfile -fastapi -greenlet -gunicorn +fastapi==0.95.1 # working 0.94.1, 0.88.0; not working >= 0.95.0, 0.95.1 +# greenlet>=2.0.2 +# gevent +gunicorn # working >=21.2.0 h11 html2text httpcore @@ -22,16 +31,16 @@ httptools httpx idna itsdangerous -Jinja2 +# Jinja2>=3.1.2 MarkupSafe mysqlclient numpy openpyxl orjson -packaging +# packaging pandas passlib -# pdf2image +pdf2image Pillow pycparser pydantic @@ -43,19 +52,20 @@ python-multipart pytz PyYAML qrcode -redis[hiredis] +redis[hiredis] # redis==5.0.0 hiredis==2.2.3 requests rfc3986 six sniffio -SQLAlchemy==1.4.47 # 1.4.47 is the newest I am working with -starlette +SQLAlchemy==1.4.52 # working 1.4.52; (2.0.29) I am working with +starlette # working ==0.26.1, 0.22.0; not working newest 0.37.2 stripe typing_extensions ujson urllib3 -uvicorn +uvicorn # working ==0.20.0 uvloop +Wand watchfiles watchgod websockets diff --git a/app/db_sql.py b/app/db_sql.py index 21b037f..1fc1b81 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -24,6 +24,7 @@ engine = create_engine(url=connection_string, poolclass=NullPool, echo=False, is # levels: "REPEATABLE READ" "READ COMMITTED" "READ UNCOMMITTED" "SERIALIZABLE" log.info('DB SQL trying to connect...') +db = None try: db = engine.connect() log.info(f'Connected to database: {db_uri}') @@ -932,6 +933,10 @@ def run_sql_select( log.debug(dir(sql)) log.debug('*** ** * ** ***') + if not db: + log.exception('The database connection is not available!!! Returning False.') + return False + try: # https://docs.sqlalchemy.org/en/13/core/tutorial.html#using-textual-sql # https://docs.sqlalchemy.org/en/13/core/sqlelement.html#sqlalchemy.sql.expression.TextClause.columns diff --git a/app/routers/api_crud.py b/app/routers/api_crud.py index adf7a93..9ebebdc 100644 --- a/app/routers/api_crud.py +++ b/app/routers/api_crud.py @@ -199,9 +199,9 @@ router = APIRouter() @router.get('/{obj_type_l1}/{obj_type_l2}/list') @router.get('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/list') async def get_obj_li( - obj_type_l1: str = Query(..., min_length=2, max_length=50), - obj_type_l2: str = Query(None, min_length=2, max_length=50), - obj_type_l3: str = Query(None, min_length=2, max_length=50), + obj_type_l1: str, + obj_type_l2: str = None, + obj_type_l3: str = None, for_obj_type: Optional[str] = Query(None, max_length=50), for_obj_id: Optional[str] = Query(None, max_length=22), @@ -621,10 +621,10 @@ async def get_obj( @router.patch('/{obj_type_l1}/{obj_type_l2}/{obj_type_l3}/{obj_id}') async def patch_obj( crud: Api_Crud_Base, - obj_type_l1: Optional[str] = Query(..., max_length=50), + obj_type_l1: str, + obj_id: str, obj_type_l2: str = None, obj_type_l3: str = None, - obj_id: str = Query(..., min_length=11, max_length=22), run_safety_check: bool = True, diff --git a/development.env b/development.env new file mode 100644 index 0000000..3305275 --- /dev/null +++ b/development.env @@ -0,0 +1,138 @@ +# One Sky IT's Aether Framework and System +TZ=US/Eastern + +CONTAINER_WEB=ae_web_dev +CONTAINER_AE_API=ae_api_dev +CONTAINER_AE_API_RED=ae_api_dev_red +CONTAINER_AE_API_GREEN=ae_api_dev_green +CONTAINER_AE_API_BLUE=ae_api_dev_blue +CONTAINER_AE_API_BLACK=ae_api_dev_black +CONTAINER_AE_API_WHITE=ae_api_dev_white +CONTAINER_AE_APP=ae_app_dev +CONTAINER_PHP7=ae_php7_dev +CONTAINER_REDIS=ae_redis_dev + +OSIT_ENV=development +# OSIT_ENV=production +# OSIT_ENV=testing + +AE_LOG_LVL=debug # Python loglevel: warning, info, debug, etc + +OSIT_WEB_HTTP_PORT=8888 +OSIT_WEB_HTTPS_PORT=443 +# Max body size is for nginx gunicorn apps (AE app and AE API) +OSIT_WEB_MAX_BODY_SIZE=5120M + +# For now this extra host variable is important for the AE Flask app to connect to the AE FastAPI API. +DOCKER_AE_SERVER_EXTRA_HOST=dev.oneskyit.com:108.28.68.107 +DOCKER_AE_APP_SERVER_EXTRA_HOST=dev-app.oneskyit.com:108.28.68.107 +DOCKER_AE_API_SERVER_EXTRA_HOST=dev-api.oneskyit.com:108.28.68.107 +DOCKER_AE_API_BAK_SERVER_EXTRA_HOST=test-api.oneskyit.com:104.237.143.4 # Odd because this env is the backup server +# DOCKER_AE_API_V5_SERVER_EXTRA_HOST=dev-api-v5.oneskyit.com:192.168.32.20 # This should be a static(ish) IP. It may need to be externally routable? +DOCKER_AE_APP_EXTRA_HOST=dev-api.oneskyit.com:108.28.68.107 +# DOCKER_AE_APP_EXTRA_HOST=default-api.oneskyit.com:104.237.143.4 +# DOCKER_AE_APP_EXTRA_HOST_V5=dev-api-v5.oneskyit.com:192.168.32.20 +DOCKER_AE_DB_SERVER_EXTRA_HOST=db.oneskyit.com:104.237.143.4 + +# This is the server name for nginx for each of these sites. +# Only one server name per server_name in nginx with envsubst!!! A bug??? +DOCKER_AE_API_SERVER_NAME=dev-api.oneskyit.com +# DOCKER_AE_API_V5_SERVER_NAME=dev-api-v5.oneskyit.com +DOCKER_AE_APP_SERVER_NAME=dev-example.oneskyit.com +DOCKER_PHPMYADMIN_SERVER_NAME=dev-phpmyadmin.oneskyit.com +DOCKER_OSIT_SERVER_NAME=dev.oneskyit.com + +# This needs to be updated for each client's subdomain. +# This is *not* currently working with the nginx Docker Compose. It uses envsubst with a template conf file. +OSIT_NGINX_SERVER_NAMES=flask_gunicorn.localhost,demo.localhost,dev.localhost,dev.oneskyit.com,dev-app.oneskyit.com,dev-connect.oneskyit.com,*.dev-connect.oneskyit.com,dev-demo.oneskyit.com,*.dev-demo.oneskyit.com,dev-aacc.oneskyit.com,*.dev-aacc.oneskyit.com,dev-aapor.oneskyit.com,*.dev-aapor.oneskyit.com,dev-businessgroup.oneskyit.com,*.dev-businessgroup.oneskyit.com,dev-chow.oneskyit.com,*.dev-chow.oneskyit.com,dev-cmsc.oneskyit.com,*.dev-cmsc.oneskyit.com,dev-idaa.oneskyit.com,*.dev-idaa.oneskyit.com,dev-ishlt.oneskyit.com,*.dev-ishlt.oneskyit.com,dev-lci.oneskyit.com,*.dev-lci.oneskyit.com,dev-ncsd.oneskyit.com,*.dev-ncsd.oneskyit.com,dev-npa.oneskyit.com,*.dev-npa.oneskyit.com,dev-rli.oneskyit.com,*.dev-rli.oneskyit.com,test-app.oneskyit.com + + +# Aether general shared config options +# For general shared config options like API access and use, database access and use, Redis, and SMTP +# home development, live testing, live production, onsite development, onsite testing, onsite production??? +AE_CFG_ID=5 + +AE_SERVER=dev.oneskyit.com + +## Aether API access and use +AE_API_PROTOCOL=https +AE_API_SERVER=dev-api.oneskyit.com +AE_API_SERVER_INTERNAL=aether_api_gunicorn +AE_API_PORT=443 +AE_API_PATH= +AE_API_SECRET_KEY=dFP6J9DVj9hUgIMn-fNIqg + +## Aether DB access and use +AE_DB_SERVER=linode.oneskyit.com +AE_DB_PORT=3306 +AE_DB_NAME=aether_dev +AE_DB_USERNAME=aether_dev +AE_DB_PASSWORD="\$1sky.AE_dev.2023" + +# wait_timeout (MariaDB) is how long to keep an idle DB connection +AE_DB_WAIT_TIMEOUT=1800 # Not yet used! +# connection_timeout (MariaDB) is how long to try and create a new DB connection; bad handshake +AE_DB_CONNECTION_TIMEOUT=20 # (in seconds; default=15) +# pool_recycle (SQLAlchemy) is how long to keep using a particular connection that has passed a certain age +AE_DB_POOL_RECYCLE=1800 + +# AE_DB_V5_SERVER=srv-nyx.oneskyit.com +# AE_DB_V5_PORT=3306 +# AE_DB_V5_NAME=aether_dev +# AE_DB_V5_USERNAME=aether_dev +# AE_DB_V5_PASSWORD="\$1sky.AE_dev.2023" + +## Aether Redis access and use +AE_REDIS_SERVER=redis +AE_REDIS_PORT=6379 + +## Aether SMTP access and use +AE_SMTP_SERVER=linode.oneskyit.com +AE_SMTP_PORT=465 +AE_SMTP_USERNAME=send_mail +# AE_SMTP_PASSWORD= + + +# Gunicorn workers and threads: +# https://docs.gunicorn.org/en/stable/design.html#how-many-workers + +# Aether API specific config options (FastAPI) +# AE_API_CFG_ID=0 # NOT CURRENTLY NEED OR USED +AE_API_ENV=development +AE_API_DIR=/srv/aether_api +AE_API_LOG_PATH="admin/logs/aether_api.log" +# AE_API_V5_LOG_PATH="/logs/aether_api_v5.log" +AE_API_GUNICORN_PORT=5065 +AE_API_GUNICORN_PORT_RED=5066 +AE_API_GUNICORN_PORT_GREEN=5067 +AE_API_GUNICORN_PORT_BLUE=5068 +AE_API_GUNICORN_PORT_BLACK=5069 +AE_API_GUNICORN_PORT_WHITE=5070 +AE_API_GUNICORN_TIMEOUT=2100 # (default=30; should be much higher; 1200 is NOT enough; worker process silent then kill and restart) +AE_API_GUNICORN_GRACEFUL_TIMEOUT=20 # (default=30; timeout after restart signal) +AE_API_GUNICORN_KEEPALIVE=5 # (default=2; was 30) +AE_API_GUNICORN_WORKERS=2 # (default=2; was 3 for a while) +AE_API_GUNICORN_THREADS=16 # (default=2) +AE_API_RELOAD=False +AE_API_JWT_KEY="EHmSXZFKfMEW65E8kxCKmQ" # 22 characters; super secret Aether JWT signing key +AE_API_ORIGINS_REGEX="(https://.*\.oneskyit\.com)|(http://.*\.oneskyit\.com)|(https://.*\.oneskyit\.com:4443)|(http://.*\.oneskyit\.com:8080)|(http://.*\.oneskyit\.com:8181)|(https://.*\.oneskyit\.com:8443)|(http://.*\.oneskyit\.local)|(http://.*\.oneskyit\.local:5000)|(http://.*.localhost)|(http://.*.localhost:5000)|(http://.*.localhost:5173)|(http://.*.localhost:8181)|(https://.*\.idaa\.org)" # default allows for some sane domains related to https://.*\.oneskyit\.com with some common extra ports + +# Aether app specific config (Flask with Svelte) +AE_APP_CFG_ID=10 +AE_APP_ENV=development +AE_APP_UX_MODE=default +# AE_APP_UX_MODE=onsite +# AE_APP_UX_MODE=native +AE_APP_DIR=/srv/aether_app +AE_APP_LOG_PATH="/logs/aether_app.log" +AE_APP_GUNICORN_PORT=5055 +AE_APP_GUNICORN_TIMEOUT=1200 # (default=30; should be higher) +AE_APP_GUNICORN_GRACEFUL_TIMEOUT=20 # (default=30) +AE_APP_GUNICORN_KEEPALIVE=90 # (default=2) +AE_APP_GUNICORN_WORKERS=1 # (default=2; was 3 for a while) +AE_APP_GUNICORN_THREADS=3 # (default=1) +AE_APP_RELOAD=True +# Generate a new key with: # python -c 'import os; print(os.urandom(16))' +AE_APP_CACHE_SECRET_KEY="$\x93\x12\xb4R\x80R\xb5\xe50\xa0k\xc8#RN" +AE_APP_SESSION_LIFETIME=172800 # How long the browser cookies last in seconds (default=86400) +AE_APP_CACHE_TIMEOUT=1 # How long the Flask app caching last in seconds (default=5)