Work towards new API version 5

aether_api_v5_fastapi_gunicorn.Dockerfile

@@ -0,0 +1,37 @@
+# FROM tiangolo/uvicorn-gunicorn-fastapi:latest
+FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10
+
+LABEL maintainer="Scott Idem <scott.idem@oneskyit.com>"
+
+WORKDIR /srv/aether_api
+
+# RUN apt install poppler-utils
+RUN apt-get update; \
+    apt-get install -y \
+        poppler-utils \
+    ; \
+    rm -rf /var/lib/apt/lists/*;
+
+# RUN set -ex; \
+#     \
+#     apt-get update; \
+#     apt-get install -y --no-install-recommends \
+#         poppler-utils \
+#     ; \
+#     rm -rf /var/lib/apt/lists/*;
+
+RUN pdftoppm -h
+
+COPY conf/aether_api_v5_fastapi_requirements.txt /tmp/requirements.txt
+
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+
+# RUN pip freeze
+# RUN mkdir /temp
+RUN pip freeze > /requirements_current.txt
+# COPY requirements_current.txt tmp/requirements_current.txt
+# RUN pip freeze > /logs/requirements_current.txt
+
+# COPY conf/aether_fastapi_requirements.txt /tmp/requirements.txt
+
+CMD ["gunicorn", "--conf", "/conf/gunicorn_fastapi_conf.py"]

conf/aether_api_v5_fastapi_config.py

@@ -0,0 +1,105 @@
+# Configuration file for this FastAPI app.
+import os
+from pydantic import AnyHttpUrl, BaseSettings, EmailStr, HttpUrl, PostgresDsn, validator
+from typing import Any, Dict, List, Optional, Union
+
+
+# ### ### #
+
+
+class Settings(BaseSettings):
+    AETHER_CFG = {}
+    AETHER_CFG['id'] = os.getenv('AE_CFG_ID', None)
+    # AETHER_CFG['api_id'] = os.getenv('AE_API_CFG_ID', None) # NOT CURRENTLY NEED OR USED
+    
+    JWT_KEY = 'EHmSXZFKfMEW65E8kxCKmQ' # 22 characters; super secret Aether JWT signing key
+
+    # APP_NAME: str = "Aether API (FastAPI)"
+    # SUPER_EMAIL: EmailStr = 'Aether.Super@oneskyit.com'
+
+
+    # Database Connection
+    DB = {}
+    DB['server'] = os.getenv('AE_DB_V5_SERVER', 'mariadb') # 'linode.oneskyit.com' # linode.oneskyit.com, vpn-linode linode.oneskyit.local
+    DB['port'] = os.getenv('AE_DB_V5_PORT', '3306') # default = 3306
+    DB['name'] = os.getenv('AE_DB_V5_NAME', None) # 'aether_dev' #onesky_ams_dev
+    DB['username'] = os.getenv('AE_DB_V5_USERNAME', None) # 'osit_aether' # 'onesky_aether'
+    DB['password'] = os.getenv('AE_DB_V5_PASSWORD', None) #
+    SQLALCHEMY_DB_URI = 'mysql://'+DB['username']+':'+DB['password']+'@'+DB['server']+'/'+DB['name']
+
+
+    # Aether API log files paths
+    LOG_PATH = {}
+    LOG_PATH['app'] = os.getenv('AE_API_V5_LOG_PATH', 'admin/log/app.log') # 'admin/log/app.log', '../../logs/aether_api.log'
+    # LOG_PATH['app_warning'] = '/logs/aether_api_warning.log' # 'admin/log/app_warning.log' '../../logs/aether_api_warning.log'
+
+
+    # Redis
+    REDIS = {}
+    REDIS['server'] = os.getenv('AE_REDIS_SERVER', 'redis') # 'localhost' 'redis'
+    REDIS['port'] = os.getenv('AE_REDIS_PORT', '6379') # '6379'
+
+
+    # Send SMTP Email
+    SMTP = {}
+    # server
+    # port
+    # username
+    # password
+
+
+    # Server Hosted File Paths
+    FILES_PATH = {}
+    # hosted_files_root
+    # hosted_tmp_root
+
+
+    # CORS Origins
+    ORIGINS_REGEX = '(https://.*\.oneskyit\.com)|(http://.*\.oneskyit\.com)|(https://.*\.oneskyit\.com:4443)|(http://.*\.oneskyit\.com:8080)|(http://.*\.oneskyit\.com:8181)|(https://.*\.oneskyit\.com:8443)|(http://.*\.oneskyit\.local)|(http://.*\.oneskyit\.local:5000)|(http://.*.localhost)|(http://.*.localhost:5000)|(http://.*.localhost:8181)'
+    # A reasonable, but fairly open example regular expression for the CORS origins:
+    # '(https://.*\.oneskyit\.com)|(http://.*\.oneskyit\.com)|(http://.*\.oneskyit\.com:8181)|(https://.*\.oneskyit\.com:8443)|(http://.*\.oneskyit\.local)|(http://.*\.oneskyit\.local:5000)|(http://.*.localhost)|(http://.*.localhost:5000)|(http://.*.localhost:8181)'
+
+    ORIGINS = [
+        'https://oneskyit.com',
+        # 'http://app-local.oneskyit.com',
+        'http://192.168.32.20:3000',
+        'http://192.168.32.20:8080',
+
+        'http://localhost',
+        'http://localhost:3000',
+        # 'http://localhost:5000',
+        'http://localhost:7800',
+        # 'http://localhost:8080',
+        # 'http://localhost:8888',
+
+        # 'http://fastapi.localhost',
+
+        'http://svelte.oneskyit.local:5555',
+
+        # 'http://connect.localhost:5000', # Using localhost
+
+        # 'http://dev-svelte.oneskyit.local:5555',
+
+        # 'http://lci.internal:5000', # Using internal; just in case guess before LCI
+        # 'http://lci.oneskyit.internal:5000', # Using internal; just in case guess before LCI
+        # 'http://lci.oneskyit.internal', # Using internal; just in case guess before LCI
+        ]
+
+
+    # HTTP Status Dict List
+    HTTP_STATUS_LI = {}
+    # HTTP_STATUS_LI[200] = { 'name': 'OK', 'message': 'The request has succeeded.' }
+    # HTTP_STATUS_LI[400] = { 'name': 'Bad Request', 'message': 'The request could not be understood by the server due to malformed syntax. The client SHOULD NOT repeat the request without modifications.' }
+    # HTTP_STATUS_LI[401] = { 'name': 'Unauthorized', 'message': 'The server could not verify that you are authorized to access the URL requested. You either supplied the wrong credentials (e.g. a bad password), or your browser does not understand how to supply the credentials required.' }
+    # HTTP_STATUS_LI[402] = { 'name': '?Request Failed?', 'message': '??The parameters were valid but the request failed.??' }
+    # HTTP_STATUS_LI[403] = { 'name': 'Forbidden', 'message': 'The server understood the request, but is refusing to fulfill it. Authorization will not help and the request SHOULD NOT be repeated. If the request method was not HEAD and the server wishes to make public why the request has not been fulfilled, it SHOULD describe the reason for the refusal in the entity. If the server does not wish to make this information available to the client, the status code 404 (Not Found) can be used instead.' }
+    # HTTP_STATUS_LI[404] = { 'name': 'Not Found', 'message': 'The requested resource does not exist.' }
+    # HTTP_STATUS_LI[409] = { 'name': 'Conflict', 'message': 'The request conflicts with another request (perhaps due to using the same idempotent key).' }
+    # HTTP_STATUS_LI[429] = { 'name': 'Too Many Requests', 'message': 'Too many requests hit the API too quickly. We recommend an exponential backoff of your requests.' }
+    # HTTP_STATUS_LI[500] = { 'name': 'Internal Server Error', 'message': 'The server encountered an unexpected condition which prevented it from fulfilling the request.' }
+    # HTTP_STATUS_LI[501] = { 'name': 'Not Implemented', 'message': 'The server does not support the functionality required to fulfill the request. This is the appropriate response when the server does not recognize the request method and is not capable of supporting it for any resource.' }
+    # HTTP_STATUS_LI[502] = { 'name': 'Bad Gateway', 'message': 'The server, while acting as a gateway or proxy, received an invalid response from the upstream server it accessed in attempting to fulfill the request.' }
+    # HTTP_STATUS_LI[503] = { 'name': 'Service Unavailable', 'message': 'The server is currently unable to handle the request due to a temporary overloading or maintenance of the server. The implication is that this is a temporary condition which will be alleviated after some delay. If known, the length of the delay MAY be indicated in a Retry-After header. If no Retry-After is given, the client SHOULD handle the response as it would for a 500 response.' }
+    # HTTP_STATUS_LI[504] = { 'name': 'Gateway Timeout', 'message': 'The server, while acting as a gateway or proxy, did not receive a timely response from the upstream server specified by the URI (e.g. HTTP, FTP, LDAP) or some other auxiliary server (e.g. DNS) it needed to access in attempting to complete the request.' }
+
+settings = Settings()

conf/aether_api_v5_fastapi_gunicorn_conf.py

@@ -0,0 +1,43 @@
+import os
+
+# Gunicorn config variables
+loglevel = "debug"
+
+accesslog = "/logs/gunicorn_access.log" # "-"  # stdout
+errorlog = "/logs/gunicorn_error.log" # "-"  # stderr
+# "logfile" does not seem to actually do anything
+# logfile = "/logs/gunicorn.log" # "-"  # stderr
+
+bind = "0.0.0.0:5005"
+# bind = "unix:/tmp/gunicorn.sock"
+
+worker_tmp_dir = "/dev/shm"
+
+chdir = "/srv/aether_api"
+# home = /path/to/environment
+wsgi_app = "app.main:app"
+# module = "run_server"
+# callable = "app"
+# plugins = "python"
+# default_proc_name = "app.main:app"
+
+# Setting a long timeout since some FastAPI API requests may take a while
+timeout = 2100 # default 30; 1200 is NOT enough; worker process silent then kill and restart
+graceful_timeout = 10 # default 30; timeout after restart signal
+keepalive = 2 # default 2; setting higher because behind load balancer (nginx)
+
+# Reload does not work correctly with UvicornWorker
+# https://github.com/benoitc/gunicorn/issues/2339
+# Disable reload if using more than one thread
+##### reload = True
+
+# reload_engine = "poll"
+
+worker_class = "uvicorn.workers.UvicornWorker" # default "sync"
+# Works are processes, not threads
+# workers = 9 # default 1; use 10ish for production; 2 to 4 times the number of cores
+# threads = 1 # default 1; only affects Gthread worker type
+workers = os.getenv('AE_API_WORKERS', 2)
+threads = os.getenv('AE_API_THREADS', 2)
+
+# umask = '007'

conf/aether_api_v5_fastapi_requirements.txt

@@ -0,0 +1,63 @@
+# aioredis # BAD! Not maintained!
+anyio
+argon2-cffi
+argon2-cffi-bindings
+asgiref
+async-timeout
+certifi
+cffi
+charset-normalizer
+click
+Deprecated
+dnspython
+email-validator
+et-xmlfile
+fastapi
+greenlet
+gunicorn
+h11
+html2text
+httpcore
+httptools
+httpx
+idna
+itsdangerous
+Jinja2
+MarkupSafe
+mysqlclient
+numpy
+openpyxl
+orjson
+packaging
+pandas
+passlib
+pdf2image
+Pillow
+pycparser
+pydantic
+PyJWT
+pyparsing
+python-dateutil
+python-dotenv
+python-multipart
+pytz
+PyYAML
+qrcode
+redis[hiredis]
+requests
+rfc3986
+six
+sniffio
+SQLAlchemy==1.4.47 # 1.4.47 is the newest I am working with
+starlette
+stripe
+typing_extensions
+ujson
+urllib3
+uvicorn
+uvloop
+watchfiles
+watchgod
+websockets
+wrapt
+xlrd

conf/nginx/site-enabled_aether_api_v5_fastapi_gunicorn.conf

@@ -0,0 +1,159 @@
+server {
+    listen 80;
+    listen [::]:80;
+    
+    server_name ${DOCKER_AE_API_V5_SERVER_NAME} v5-fastapi.localhost api-v5.localhost;
+    # server_name
+    #     fastapi_gunicorn.localhost
+    #     dev-api.localhost
+    #     dev-api.oneskyit.com
+    #     test-api.oneskyit.com
+    #     ;
+
+    access_log  /logs/nginx/access_fastapi_gunicorn.log;
+    error_log  /logs/nginx/error_fastapi_gunicorn.log;
+
+    client_max_body_size 5120M; #4096M or 4G; 5120M or 5G;
+
+    location / {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        # This is needed for long running Python code. Default is 60 seconds
+        # Increased from 1200 to 1500 on 2022-04-17
+        # Increased from 1500 to 2000 on 2023-03-15
+        # Increased proxy read timeout to 2100 and decreased fastcgi options to 35s on 2023-03-16
+        fastcgi_connect_timeout 4s;
+        fastcgi_send_timeout 5s;
+        fastcgi_read_timeout 5s;
+        
+        # proxy read timeout being too low will cause 504 Gateway Time-out on the client browser
+        proxy_read_timeout 2100s;
+
+        proxy_pass http://v5_fastapi_backend;
+    }
+    
+    location /ws {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_http_version 1.1;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        # proxy_read_timeout 600;
+        # proxy_headers_hash_max_size 1024;
+
+        proxy_pass http://v5_fastapi_backend;
+        
+        access_log  /logs/nginx/access_fastapi_gunicorn_ws.log;
+        error_log  /logs/nginx/error_fastapi_gunicorn_ws.log;
+    }
+}
+
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl http2;
+    
+    server_name ${DOCKER_AE_API_V5_SERVER_NAME} v5-fastapi.localhost api-v5.localhost;
+    # server_name
+    #     fastapi_gunicorn.localhost
+    #     dev-api.localhost
+    #     dev-api.oneskyit.com
+    #     test-api.oneskyit.com
+    #     ;
+
+    access_log  /logs/nginx/access_fastapi_gunicorn.log;
+    error_log  /logs/nginx/error_fastapi_gunicorn.log;
+
+    include /etc/nginx/options-ssl-nginx.conf;
+
+    ssl_certificate /etc/certs/fullchain.pem;
+    ssl_certificate_key /etc/certs/privkey.pem;
+    ssl_dhparam /etc/certs/ssl-dhparams.pem;
+
+    # include brotli.conf;
+    # include gzip.conf;
+
+    client_max_body_size 5120M; #4096M or 4G; 5120M or 5G;
+
+    location / {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        # This is needed for long running Python code. Default is 60 seconds
+        # Increased from 1200 to 1500 on 2022-04-17
+        # Increased from 1500 to 2000 on 2023-03-15
+        # Increased proxy read timeout to 2100 and decreased fastcgi options to 35s on 2023-03-16
+        fastcgi_connect_timeout 35s;
+        fastcgi_send_timeout 35s;
+        fastcgi_read_timeout 35s;
+        
+        # proxy read timeout being too low will cause 504 Gateway Time-out on the client browser
+        proxy_read_timeout 2100s;
+
+        proxy_pass http://v5_fastapi_backend;
+    }
+    
+    location /ws {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_http_version 1.1;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        # proxy_read_timeout 600;
+        # proxy_headers_hash_max_size 1024;
+
+        proxy_pass http://v5_fastapi_backend;
+        
+        access_log  /logs/nginx/access_fastapi_gunicorn_ws.log;
+        error_log  /logs/nginx/error_fastapi_gunicorn_ws.log;
+    }
+}
+
+
+upstream v5_fastapi_backend {
+    # sticky sessions
+    ip_hash;
+
+    # enable least connections balancing method
+    # least_conn;
+
+    # zone backend 64k;  # Use NGINX Plus' shared memory
+
+    # server webserver1 weight=1;
+    # server webserver2 weight=4;
+
+    # larger number will recieve more requests
+    # Example of 20 vs 10: 20 will recieve twice as many requests as 10
+    server aether_api_v5_gunicorn:5005 weight=20 max_fails=3 fail_timeout=30s;
+    # server aether_api_gunicorn_bak:5005 weight=10 max_fails=1 fail_timeout=30s;
+
+    # maintain up to 20 idle connections to the group of upstream servers
+    # keepalive 20;
+}

conf/nginx/site-enabled_aether_flask_gunicorn.conf

@@ -12,6 +12,7 @@ server {
         dev-connect.oneskyit.com *.dev-connect.oneskyit.com
         dev-demo.oneskyit.com *.dev-demo.oneskyit.com
         
+        dev-aacc.oneskyit.com *.dev-aacc.oneskyit.com
         dev-aapor.oneskyit.com *.dev-aapor.oneskyit.com
         dev-businessgroup.oneskyit.com *.dev-businessgroup.oneskyit.com
         
@@ -68,6 +69,7 @@ server {
         dev-connect.oneskyit.com *.dev-connect.oneskyit.com
         dev-demo.oneskyit.com *.dev-demo.oneskyit.com
         
+        dev-aacc.oneskyit.com *.dev-aacc.oneskyit.com
         dev-aapor.oneskyit.com *.dev-aapor.oneskyit.com
         dev-businessgroup.oneskyit.com *.dev-businessgroup.oneskyit.com
         

docker-compose.yml

@@ -17,7 +17,7 @@ services:
             # - NGINX_SERVER_NAMES="dev-idaa.oneskyit.com dev-ishlt.oneskyit.com"
             # - AE_DOMAIN_LIST:'dev-aapor.oneskyit.com dev-businessgroup.oneskyt.com dev-cmsc.oneskyit.com dev-idaa.oneskyit.com dev-ishlt.oneskyit.com dev-ncsd.oneskyit.com dev-npa.oneskyit.com dev-rli.oneskyit.com'
             # - NGINX_SERVER_NAMES="flask_gunicorn.localhost demo.localhost dev.localhost dev.oneskyit.com dev-app.oneskyit.com dev-connect.oneskyit.com *.dev-connect.oneskyit.com dev-demo.oneskyit.com *.dev-demo.oneskyit.com dev-aapor.oneskyit.com *.dev-aapor.oneskyit.com dev-businessgroup.oneskyt.com *.dev-businessgroup.oneskyt.com dev-cmsc.oneskyit.com *.dev-cmsc.oneskyit.com dev-idaa.oneskyit.com *.dev-idaa.oneskyit.com dev-ishlt.oneskyit.com *.dev-ishlt.oneskyit.com dev-ncsd.oneskyit.com *.dev-ncsd.oneskyit.com dev-npa.oneskyit.com *.dev-npa.oneskyit.com dev-rli.oneskyit.com *.dev-rli.oneskyit.com test-app.oneskyit.com"
-            - NGINX_SERVER_NAMES="flask_gunicorn.localhost demo.localhost dev.localhost dev.oneskyit.com dev-app.oneskyit.com dev-connect.oneskyit.com dev-demo.oneskyit.com dev-aapor.oneskyit.com dev-businessgroup.oneskyt.com dev-cmsc.oneskyit.com dev-idaa.oneskyit.com dev-ishlt.oneskyit.com dev-ncsd.oneskyit.com dev-npa.oneskyit.com dev-rli.oneskyit.com test-app.oneskyit.com"
+            - NGINX_SERVER_NAMES="flask_gunicorn.localhost demo.localhost dev.localhost dev.oneskyit.com dev-app.oneskyit.com dev-connect.oneskyit.com dev-demo.oneskyit.com dev-aacc.oneskyit.com dev-aapor.oneskyit.com dev-businessgroup.oneskyt.com dev-cmsc.oneskyit.com dev-idaa.oneskyit.com dev-ishlt.oneskyit.com dev-ncsd.oneskyit.com dev-npa.oneskyit.com dev-rli.oneskyit.com test-app.oneskyit.com"
         ports:
             - "${OSIT_WEB_HTTP_PORT}:80"
             - "${OSIT_WEB_HTTPS_PORT}:443"
@@ -56,6 +56,7 @@ services:
             - ./conf/nginx/site-enabled_oneskyit.conf:/etc/nginx/templates/site-enabled_oneskyit.conf.template
 
             - ./conf/nginx/site-enabled_aether_fastapi_gunicorn.conf:/etc/nginx/templates/site-enabled_aether_fastapi_gunicorn.conf.template
+            - ./conf/nginx/site-enabled_aether_api_v5_fastapi_gunicorn.conf:/etc/nginx/templates/site-enabled_aether_api_v5_fastapi_gunicorn.conf.template
             - ./conf/nginx/site-enabled_aether_flask_gunicorn.conf:/etc/nginx/templates/site-enabled_aether_flask_gunicorn.conf.template
 
             - ./conf/certs/fullchain.pem:/etc/certs/fullchain.pem
@@ -68,6 +69,7 @@ services:
         depends_on:
             - php7
             - aether_api_gunicorn
+            - aether_api_v5_gunicorn
             - aether_app_gunicorn
         # links:
         #     # - nextcloud25
@@ -277,6 +279,39 @@ services:
             - redis
         stdin_open: true # docker run -i
         tty: true        # docker run -t
+        
+    aether_api_v5_gunicorn:
+        restart: always
+        container_name: ae_api_v5_dev
+        build:
+            # context: ./builds
+            context: ./
+            dockerfile: aether_api_v5_fastapi_gunicorn.Dockerfile
+        env_file:
+            - ./.env
+        ports:
+            - "5067:5005"
+        volumes:
+            - ./conf/aether_api_v5_fastapi_gunicorn_conf.py:/conf/gunicorn_fastapi_conf.py
+            - ./conf/aether_api_v5_fastapi_requirements_current.txt:/conf/requirements_current.txt
+            - ./conf/aether_api_v5_fastapi_config.py:/srv/aether_api/app/config.py
+            
+            - ./logs/ae_api_v5:/logs
+        
+            - ./srv/aether_api_v5_ln:/srv/aether_api
+            - ./srv/hosted_files_ln:/srv/hosted_files
+            - ./srv/hosted_tmp_ln:/srv/hosted_tmp
+            
+            # - ./tmp/ae_api:/tmp
+            
+            - ./temp/ae_api:/temp
+            # - ./temp/ae_api/aether_fastapi_requirements_current.txt:/temp/requirements_current.txt
+        # links:
+        #     - redis
+        depends_on:
+            - redis
+        stdin_open: true # docker run -i
+        tty: true        # docker run -t
 
     aether_app_gunicorn:
         restart: always
@@ -297,6 +332,7 @@ services:
         extra_hosts:
              # - dev-api.oneskyit.com:192.168.32.20
              - "${DOCKER_AE_APP_EXTRA_HOST}"
+             - "${DOCKER_AE_APP_EXTRA_HOST_V5}"
         volumes:
             - ./conf/aether_flask_gunicorn_conf.py:/conf/gunicorn_flask_conf.py
             - ./conf/aether_app_config.py:/srv/aether_app/flask_config_v2.py

logs/ae_api_v5/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except for this file
+!.gitignore

srv/aether_api_v5_ln

@@ -0,0 +1 @@
+/home/scott/OSIT_dev/aether_api_v5_fastapi