General updates and clean up.

aether_fastapi_gunicorn.Dockerfile

@@ -5,7 +5,33 @@ LABEL maintainer="Scott Idem <scott.idem@oneskyit.com>"
 
 WORKDIR /srv/aether_api
 
+# RUN apt install poppler-utils
+RUN apt-get update; \
+    apt-get install -y \
+        poppler-utils \
+    ; \
+    rm -rf /var/lib/apt/lists/*;
+
+# RUN set -ex; \
+#     \
+#     apt-get update; \
+#     apt-get install -y --no-install-recommends \
+#         poppler-utils \
+#     ; \
+#     rm -rf /var/lib/apt/lists/*;
+
+RUN pdftoppm -h
+
 COPY conf/aether_fastapi_requirements.txt /tmp/requirements.txt
+
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
+# RUN pip freeze
+# RUN mkdir /temp
+RUN pip freeze > /requirements_current.txt
+# COPY requirements_current.txt tmp/requirements_current.txt
+# RUN pip freeze > /logs/requirements_current.txt
+
+# COPY conf/aether_fastapi_requirements.txt /tmp/requirements.txt
+
 CMD ["gunicorn", "--conf", "/conf/gunicorn_fastapi_conf.py"]

conf/aether_api_config.py

@@ -11,7 +11,8 @@ class Settings(BaseSettings):
     AETHER_CFG = {}
     AETHER_CFG['id'] = os.getenv('AE_CFG_ID', None)
     # AETHER_CFG['api_id'] = os.getenv('AE_API_CFG_ID', None) # NOT CURRENTLY NEED OR USED
-
+    
+    JWT_KEY = 'EHmSXZFKfMEW65E8kxCKmQ' # 22 characters; super secret Aether JWT signing key
 
     # APP_NAME: str = "Aether API (FastAPI)"
     # SUPER_EMAIL: EmailStr = 'Aether.Super@oneskyit.com'

conf/aether_fastapi_gunicorn_conf.py

@@ -22,9 +22,9 @@ wsgi_app = "app.main:app"
 # default_proc_name = "app.main:app"
 
 # Setting a long timeout since some FastAPI API requests may take a while
-timeout = 1200 # default 30; worker process silent then kill and restart
+timeout = 2100 # default 30; 1200 is NOT enough; worker process silent then kill and restart
-graceful_timeout = 20 # default 30; timeout after restart signal
+graceful_timeout = 10 # default 30; timeout after restart signal
-keepalive = 300 # default 2; setting higher because behind load balancer (nginx)
+keepalive = 2 # default 2; setting higher because behind load balancer (nginx)
 
 # Reload does not work correctly with UvicornWorker
 # https://github.com/benoitc/gunicorn/issues/2339
@@ -37,7 +37,7 @@ worker_class = "uvicorn.workers.UvicornWorker" # default "sync"
 # Works are processes, not threads
 # workers = 9 # default 1; use 10ish for production; 2 to 4 times the number of cores
 # threads = 1 # default 1; only affects Gthread worker type
-workers = os.getenv('AE_API_WORKERS', None)
+workers = os.getenv('AE_API_WORKERS', 2)
-threads = os.getenv('AE_API_THREADS', None)
+threads = os.getenv('AE_API_THREADS', 2)
 
 # umask = '007'

conf/aether_fastapi_requirements.txt

@@ -1,61 +1,61 @@
-anyio==3.6.2
+anyio
-argon2-cffi==21.3.0
+argon2-cffi
-argon2-cffi-bindings==21.2.0
+argon2-cffi-bindings
-asgiref==3.5.2
+asgiref
-async-timeout==4.0.2
+async-timeout
-certifi==2022.9.24
+certifi
-cffi==1.15.1
+cffi
-charset-normalizer==2.1.1
+charset-normalizer
-click==8.1.3
+click
-Deprecated==1.2.13
+Deprecated
-dnspython==2.2.1
+dnspython
-email-validator==1.3.0
+email-validator
-et-xmlfile==1.1.0
+et-xmlfile
-fastapi==0.88.0
+fastapi
-greenlet==2.0.0
+greenlet
-gunicorn==20.1.0
+gunicorn
-h11==0.14.0
+h11
-html2text==2020.1.16
+html2text
-httpcore==0.16.2
+httpcore
-httptools==0.5.0
+httptools
-httpx==0.23.1
+httpx
-idna==3.4
+idna
-itsdangerous==2.1.2
+itsdangerous
-Jinja2==3.1.2
+Jinja2
-MarkupSafe==2.1.1
+MarkupSafe
-mysqlclient==2.1.1
+mysqlclient
-numpy==1.23.4
+numpy
-openpyxl==3.0.10
+openpyxl
-orjson==3.8.1
+orjson
-packaging==21.3
+packaging
-pandas==1.5.2
+pandas
-passlib==1.7.4
+passlib
-Pillow==9.3.0
+Pillow
-pycparser==2.21
+pycparser
-pydantic==1.10.2
+pydantic
-PyJWT==2.6.0
+PyJWT
-pyparsing==3.0.9
+pyparsing
-python-dateutil==2.8.2
+python-dateutil
-python-dotenv==0.21.0
+python-dotenv
-python-multipart==0.0.5
+python-multipart
-pytz==2022.6
+pytz
-PyYAML==6.0
+PyYAML
-qrcode==7.3.1
+qrcode
-redis==4.3.5
+redis
-requests==2.28.1
+requests
-rfc3986==1.5.0
+rfc3986
-six==1.16.0
+six
-sniffio==1.3.0
+sniffio
-SQLAlchemy==1.4.44
+SQLAlchemy==1.4.47 # 1.4.47 is the newest I am working with
-starlette==0.22.0
+starlette
-stripe==5.0.0
+stripe
-typing_extensions==4.4.0
+typing_extensions
-ujson==5.5.0
+ujson
-urllib3==1.26.12
+urllib3
-uvicorn==0.18.3
+uvicorn
-uvloop==0.17.0
+uvloop
-watchfiles==0.18.0
+watchfiles
-watchgod==0.8.2
+watchgod
-websockets==10.4
+websockets
-wrapt==1.14.1
+wrapt
-xlrd==2.0.1
+xlrd

conf/aether_flask_gunicorn_conf.py

@@ -33,8 +33,8 @@ reload = True
 # Works are processes, not threads
 # workers = 9 # default 1; use 10ish for production; 2 to 4 times the number of cores
 # threads = 1 # default 1; only affects Gthread worker type
-workers = os.getenv('AE_APP_WORKERS', None)
+workers = os.getenv('AE_APP_WORKERS', 2)
-threads = os.getenv('AE_APP_THREADS', None)
+threads = os.getenv('AE_APP_THREADS', 2)
 
 # umask = '007'
 

conf/nginx/site-enabled_aether_fastapi_gunicorn.conf

@@ -2,7 +2,7 @@ server {
     listen 80;
     listen [::]:80;
     
-    server_name ${DOCKER_AE_API_SERVER_NAME};
+    server_name ${DOCKER_AE_API_SERVER_NAME} fastapi.localhost api.localhost;
     # server_name
     #     fastapi_gunicorn.localhost
     #     dev-api.localhost
@@ -26,13 +26,40 @@ server {
 
         # This is needed for long running Python code. Default is 60 seconds
         # Increased from 1200 to 1500 on 2022-04-17
-        fastcgi_connect_timeout 1500s;
+        # Increased from 1500 to 2000 on 2023-03-15
-        fastcgi_send_timeout 1500s;
+        # Increased proxy read timeout to 2100 and decreased fastcgi options to 35s on 2023-03-16
-        fastcgi_read_timeout 1500s;
+        fastcgi_connect_timeout 4s;
-        proxy_read_timeout 1500s;
+        fastcgi_send_timeout 5s;
+        fastcgi_read_timeout 5s;
+        
+        # proxy read timeout being too low will cause 504 Gateway Time-out on the client browser
+        proxy_read_timeout 2100s;
 
         proxy_pass http://fastapi_backend;
     }
+    
+    location /ws {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_http_version 1.1;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        # proxy_read_timeout 600;
+        # proxy_headers_hash_max_size 1024;
+
+        proxy_pass http://fastapi_backend;
+        
+        access_log  /logs/nginx/access_fastapi_gunicorn_ws.log;
+        error_log  /logs/nginx/error_fastapi_gunicorn_ws.log;
+    }
 }
 
 
@@ -40,7 +67,7 @@ server {
     listen 443 ssl;
     listen [::]:443 ssl http2;
     
-    server_name ${DOCKER_AE_API_SERVER_NAME};
+    server_name ${DOCKER_AE_API_SERVER_NAME} fastapi.localhost api.localhost;
     # server_name
     #     fastapi_gunicorn.localhost
     #     dev-api.localhost
@@ -73,13 +100,40 @@ server {
 
         # This is needed for long running Python code. Default is 60 seconds
         # Increased from 1200 to 1500 on 2022-04-17
-        fastcgi_connect_timeout 1500s;
+        # Increased from 1500 to 2000 on 2023-03-15
-        fastcgi_send_timeout 1500s;
+        # Increased proxy read timeout to 2100 and decreased fastcgi options to 35s on 2023-03-16
-        fastcgi_read_timeout 1500s;
+        fastcgi_connect_timeout 35s;
-        proxy_read_timeout 1500s;
+        fastcgi_send_timeout 35s;
+        fastcgi_read_timeout 35s;
+        
+        # proxy read timeout being too low will cause 504 Gateway Time-out on the client browser
+        proxy_read_timeout 2100s;
 
         proxy_pass http://fastapi_backend;
     }
+    
+    location /ws {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_http_version 1.1;
+
+        proxy_redirect off;
+        proxy_buffering off;
+
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        # proxy_read_timeout 600;
+        # proxy_headers_hash_max_size 1024;
+
+        proxy_pass http://fastapi_backend;
+        
+        access_log  /logs/nginx/access_fastapi_gunicorn_ws.log;
+        error_log  /logs/nginx/error_fastapi_gunicorn_ws.log;
+    }
 }
 
 

conf/nginx/site-enabled_aether_flask_gunicorn.conf

@@ -13,7 +13,7 @@ server {
         dev-demo.oneskyit.com *.dev-demo.oneskyit.com
         
         dev-aapor.oneskyit.com *.dev-aapor.oneskyit.com
-        dev-businessgroup.oneskyt.com *.dev-businessgroup.oneskyt.com
+        dev-businessgroup.oneskyit.com *.dev-businessgroup.oneskyit.com
         
         dev-cmsc.oneskyit.com *.dev-cmsc.oneskyit.com
         dev-idaa.oneskyit.com *.dev-idaa.oneskyit.com
@@ -69,7 +69,7 @@ server {
         dev-demo.oneskyit.com *.dev-demo.oneskyit.com
         
         dev-aapor.oneskyit.com *.dev-aapor.oneskyit.com
-        dev-businessgroup.oneskyt.com *.dev-businessgroup.oneskyt.com
+        dev-businessgroup.oneskyit.com *.dev-businessgroup.oneskyit.com
         
         dev-cmsc.oneskyit.com *.dev-cmsc.oneskyit.com
         dev-idaa.oneskyit.com *.dev-idaa.oneskyit.com

docker-compose.yml

@@ -223,6 +223,7 @@ services:
     #         - NEXTCLOUD_TRUSTED_DOMAINS=oneskyit.com
 
     redis:
+        restart: always
         container_name: ae_redis_dev
         image: redis
         ports:
@@ -247,6 +248,7 @@ services:
         #   - local-net
         volumes:
             - ./conf/aether_fastapi_gunicorn_conf.py:/conf/gunicorn_fastapi_conf.py
+            - ./conf/aether_fastapi_requirements_current.txt:/conf/requirements_current.txt
             - ./conf/aether_api_config.py:/srv/aether_api/app/config.py
             
             - ./logs/ae_api:/logs
@@ -264,6 +266,11 @@ services:
             - ./srv/aether_api_ln:/srv/aether_api
             - ./srv/hosted_files_ln:/srv/hosted_files
             - ./srv/hosted_tmp_ln:/srv/hosted_tmp
+            
+            # - ./tmp/ae_api:/tmp
+            
+            - ./temp/ae_api:/temp
+            # - ./temp/ae_api/aether_fastapi_requirements_current.txt:/temp/requirements_current.txt
         # links:
         #     - redis
         depends_on: