Fix Bitbucket auth migration in deploy workflow

Bind Dozzle to 127.0.0.1 to prevent exposure on external/LAN interfaces.
Previously bound to 0.0.0.0, allowing unauthenticated access to container
logs from any network-reachable host.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.env.default

@@ -79,9 +79,27 @@ AE_DB_USERNAME=aether_dev
 AE_DB_PASSWORD=XXXX
 
 # Connection Tuning
+# Seconds to wait when establishing a new connection before giving up.
+# Lower values fail fast on DB outage rather than hanging requests.
 AE_DB_CONNECTION_TIMEOUT=7
+
+# Seconds before a pooled connection is recycled (closed and reopened).
+# Prevents "MySQL server has gone away" errors from MariaDB's wait_timeout.
+# Must be less than MariaDB's wait_timeout (default 28800s / 8 hours).
+# 900s (15 min) is a safe conservative value for active workloads.
 AE_DB_POOL_RECYCLE=900
 
+# Connections held open per API replica at idle (the "warm" pool).
+# Each replica maintains this many persistent connections to MariaDB.
+AE_DB_POOL_SIZE=10
+
+# Additional connections a replica can open beyond AE_DB_POOL_SIZE under burst load.
+# These are created on demand and closed when the burst subsides.
+# Max connections per replica = AE_DB_POOL_SIZE + AE_DB_POOL_MAX_OVERFLOW.
+# Total max DB connections across all replicas = AE_API_REPLICAS × (AE_DB_POOL_SIZE + AE_DB_POOL_MAX_OVERFLOW).
+# Example: 3 replicas × (10 + 20) = 90 max connections. MARIADB_MAX_CONNECTIONS must exceed this.
+AE_DB_POOL_MAX_OVERFLOW=20
+
 # ------------------------------------------------------------------------------
 # REDIS SETTINGS
 # ------------------------------------------------------------------------------
@@ -92,23 +110,44 @@ AE_REDIS_PORT=6379
 # ------------------------------------------------------------------------------
 # API SETTINGS (FastAPI)
 # ------------------------------------------------------------------------------
-# Number of API container instances to run (Docker Compose scaling)
+
+# Number of API container instances (Docker Compose replica scaling).
+# Each replica is an independent container with its own Gunicorn process and
+# connection pool. Total DB connections = AE_API_REPLICAS × (AE_DB_POOL_SIZE + AE_DB_POOL_MAX_OVERFLOW).
+# Increase for horizontal scaling across CPU cores. On a single-node Linode,
+# 2-4 replicas is typical; more replicas won't help if the DB is the bottleneck.
 AE_API_REPLICAS=3
 
-# Gunicorn / Uvicorn Tuning
-# AE_API_GUNICORN_TIMEOUT: worker timeout in seconds. Default in gunicorn_conf.py
-# is 120s. Raise for endpoints that run long ffmpeg operations (clip_video, etc.)
-# The dev .env typically sets this to 900 to accommodate 5-15 min video jobs.
+# --- Gunicorn / Uvicorn Tuning ---
+
+# Internal port Gunicorn listens on inside the container. Nginx proxies to this.
+# Each replica uses this same port within its own network namespace.
 AE_API_GUNICORN_PORT=5065
+
+# Worker timeout in seconds. A request that takes longer than this causes Gunicorn
+# to kill and restart the worker. Default in gunicorn_conf.py is 120s.
+# Raise for endpoints that run long ffmpeg operations (clip_video, convert_file, etc.).
+# Dev typically uses 900s to accommodate 5-15 min video jobs.
 AE_API_GUNICORN_TIMEOUT=900
-AE_API_GUNICORN_WORKERS=2
+
+# Uvicorn worker processes per replica. Each worker handles requests independently
+# using async I/O, but SQLAlchemy DB calls are synchronous and block the worker.
+# More workers = more parallel DB queries. Recommended: 2-4 per replica.
+# Total parallel DB query capacity ≈ AE_API_REPLICAS × AE_API_GUNICORN_WORKERS.
+# Stress testing at 4 workers/replica yielded ~2x throughput vs 2 workers (14 req/s vs 7.5 req/s).
+# Rule of thumb: (2 × CPU cores) + 1 per replica, but DB throughput caps before CPU becomes the limit.
+AE_API_GUNICORN_WORKERS=4
+
+# Threads per Gunicorn worker. Uvicorn workers use async I/O, so threading provides
+# minimal benefit here. Leave at 1 unless explicitly benchmarked otherwise.
 AE_API_GUNICORN_THREADS=1
 
 # Security & CORS
-# JWT_KEY should be a 22+ character secret string
+# JWT_KEY should be a 22+ character secret string. Rotate if compromised.
 AE_API_JWT_KEY=XXXX
 
-# Regex for allowed CORS origins
+# Regex for allowed CORS origins. Requests from non-matching origins are blocked.
+# Extend the pattern if adding new domains or local dev ports.
 AE_API_ORIGINS_REGEX="(https://.*\.oneskyit\.com)|(http://.*\.oneskyit\.com)|(http://.*.localhost)|(http://.*.localhost:5173)"
 
 # ------------------------------------------------------------------------------

README.md

@@ -23,6 +23,7 @@ workstation:3001           workstation:5060
 ```
 
 **Key Improvements:**
+- **Timezone Support:** All containers use the `TZ` variable from `.env` for consistent logging and database timestamps.
 - **Scalable Routing:** Nginx uses Regex (`~^(dev|test|bak|sr|prod)?-?...`) to automatically handle any environment prefix without configuration changes.
 - **Isolated Stacks:** Each deployment uses a unique `AE_NETWORK_NAME` and `CONTAINER_` prefix to prevent collisions.
 - **Shared Services:** Core infrastructure (DB/Redis) resides on the `aether_shared_net` which must be created manually once.
@@ -42,7 +43,7 @@ Create the base directory and clone this environment:
 ```bash
 sudo mkdir -p /srv/env/aether
 sudo chown -R $USER:$USER /srv/env/aether
-git clone https://bitbucket.org/oneskyit/one-sky-it-container-environment.git /srv/env/aether/container_env
+git clone git@bitbucket.org:oneskyit/one-sky-it-container-environment.git /srv/env/aether/container_env
 ```
 
 ### 3. Configure Environment Settings
@@ -119,7 +120,7 @@ These scripts are located in the root directory:
 ## 📂 Directory Map
 
 *   **`conf/`**: Configuration templates for Nginx and Gunicorn. API config now lives in the `aether_api_fastapi` repo as `app/config.py` and reads settings directly from env vars.
-*   **`logs/`**: Centralized logging for all containers.
+*   **`logs/`**: Centralized logging for all containers. Automatic rotation is managed by the `ae_ops` service (7-day retention).
 *   **`srv/`**: Mount points for data and source code (managed via symlinks).
 *   **`scripts/`**: Internal automation logic.
 *   **`backups/`**: Storage for MariaDB snapshots.

conf/crontab

@@ -1 +1,2 @@
 55 * * * * bash /scripts/backup_internal.sh >> /logs/backup_cron.log 2>&1
+0 0 * * * /usr/sbin/logrotate /etc/logrotate.internal.conf

conf/logrotate.conf

@@ -1,7 +1,7 @@
-# Logrotate configuration for Aether Docker Logs
-# To use: sudo ln -s /home/scott/OSIT_dev/aether_container_env/conf/logrotate.conf /etc/logrotate.d/aether
+# Logrotate configuration for Aether Docker Logs (Internal container version)
 
-/home/scott/OSIT_dev/aether_container_env/logs/*/*.log {
+/logs/*/*.log /logs/web/*/*.log {
+    su aether aether
     daily
     rotate 7
     missingok

deploy.sh

@@ -16,6 +16,18 @@
 
 set -euo pipefail
 
+ensure_bitbucket_ssh_remote() {
+    local repo_path=$1
+    local remote_url
+    local remote_path
+
+    remote_url=$(git -C "$repo_path" remote get-url origin)
+    if [[ "$remote_url" =~ ^https://([^@/]+@)?bitbucket\.org/(.+)$ ]]; then
+        remote_path=${BASH_REMATCH[2]}
+        git -C "$repo_path" remote set-url origin "git@bitbucket.org:${remote_path}"
+    fi
+}
+
 ENV=${1:-}
 if [ -z "$ENV" ]; then
     echo "Usage: $0 <prod|test> [app_branch] [api_branch]"
@@ -55,14 +67,17 @@ echo ""
 
 # --- Pull repos ---
 echo "[1/4] Pulling container env..."
+ensure_bitbucket_ssh_remote "$COMPOSE_DIR"
 git -C "$COMPOSE_DIR" pull --ff-only
 
 echo ""
 echo "[2/4] Pulling app ($APP_BRANCH)..."
+ensure_bitbucket_ssh_remote "$APP_DIR"
 git -C "$APP_DIR" pull --ff-only origin "$APP_BRANCH"
 
 echo ""
 echo "[3/4] Pulling API ($API_BRANCH)..."
+ensure_bitbucket_ssh_remote "$API_DIR"
 git -C "$API_DIR" pull --ff-only origin "$API_BRANCH"
 
 # --- Build and deploy ---

docker-compose.yml

@@ -13,7 +13,7 @@ services:
         environment:
             - PUID=1000
             - PGID=1000
-            - TZ=US/Eastern
+            - TZ=${TZ}
 
             - NGINX_SERVER_NAMES="demo.localhost dev.localhost dev.oneskyit.com dev-app.oneskyit.com dev-connect.oneskyit.com dev-demo.oneskyit.com dev-aacc.oneskyit.com dev-aapor.oneskyit.com dev-ascm.oneskyit.com dev-businessgroup.oneskyt.com dev-chow.oneskyit.com dev-cmsc.oneskyit.com dev-idaa.oneskyit.com dev-ishlt.oneskyit.com dev-lci.oneskyit.com dev-ncsd.oneskyit.com dev-npa.oneskyit.com dev-rli.oneskyit.com test-app.oneskyit.com test-api.oneskyit.com test-demo.oneskyit.com test-lci.oneskyit.com test-idaa.oneskyit.com scott.oneskyit.com dgr.oneskyit.com"
         ports:
@@ -30,7 +30,6 @@ services:
             - ./conf/nginx/site.conf:/etc/nginx/conf.d/0_site.conf
             - ./conf/nginx/site-enabled_aether_fastapi_gunicorn.conf:/etc/nginx/templates/site-enabled_aether_fastapi_gunicorn.conf.template
             - ./conf/nginx/site-enabled_aether_app_svelte_node.conf:/etc/nginx/templates/site-enabled_aether_app_svelte_node.conf.template
-            # - ./conf/nginx/site-enabled_aether_flask_gunicorn.conf:/etc/nginx/templates/site-enabled_aether_flask_gunicorn.conf.template
             - ./conf/certs/oneskyit_wild_fullchain.pem:/etc/certs/fullchain_wild.pem
             - ./conf/certs/oneskyit_wild_privkey.pem:/etc/certs/privkey_wild.pem
             - ./conf/certs/oneskyit.com_fullchain.pem:/etc/certs/fullchain.pem
@@ -40,7 +39,6 @@ services:
         depends_on:
             - ae_api
             - ae_app
-            # - aether_app_gunicorn
         logging:
             driver: "json-file"
             options:
@@ -54,6 +52,8 @@ services:
         networks:
             - default
             - shared
+        environment:
+            - TZ=${TZ}
         command: redis-server --save "" --loglevel warning
         logging:
             driver: "json-file"
@@ -82,6 +82,7 @@ services:
             MYSQL_DATABASE: ${AE_DB_NAME}
             MYSQL_USER: ${AE_DB_USERNAME}
             MYSQL_PASSWORD: ${AE_DB_PASSWORD}
+            TZ: ${TZ}
         ports:
             - "${AE_DB_EXTERNAL_PORT}:3306"
         volumes:
@@ -103,6 +104,7 @@ services:
         environment:
             PMA_HOST: mariadb
             UPLOAD_LIMIT: 64M
+            TZ: ${TZ}
         ports:
             - "${AE_PMA_PORT}:80"
         depends_on:
@@ -180,10 +182,8 @@ services:
             home.oneskyit.com: "71.126.159.102"
             static.oneskyit.com: "104.237.143.4"
             dev.oneskyit.com: "192.168.32.7"
-        # volumes:
-        #    # In production, the build happens INSIDE the container.
-        #    # Mounting the host source here would override the internal build.
-        #    # - ${AE_APP_SRC}:/app
+        volumes:
+            - ./logs/ae_app:/logs
         depends_on:
             - ae_api
             - redis
@@ -197,10 +197,12 @@ services:
     dozzle:
         container_name: ${CONTAINER_DOZZLE:-ae_dozzle_dev}
         image: amir20/dozzle:latest
+        environment:
+            - TZ=${TZ}
         volumes:
             - /var/run/docker.sock:/var/run/docker.sock
         ports:
-            - "${AE_DOZZLE_PORT:-8881}:8080"
+            - "127.0.0.1:${AE_DOZZLE_PORT:-8881}:8080"
         restart: unless-stopped
         logging:
             driver: "json-file"
@@ -209,7 +211,6 @@ services:
                 max-file: "3"
 
     ae_ops:
-        # ... (same as before) ...
         container_name: ${CONTAINER_AE_OPS:-ae_ops_dev}
         image: alpine:latest
         restart: always
@@ -224,7 +225,8 @@ services:
             - ./scripts:/scripts
             - ./logs:/logs
             - ./conf/crontab:/etc/crontabs/root
-        command: sh -c "apk add --no-cache docker-cli bash && crond -f -l 2"
+            - ./conf/logrotate.conf:/etc/logrotate.conf
+        command: sh -c "apk add --no-cache docker-cli bash logrotate && adduser -u 1000 -D aether && cp /etc/logrotate.conf /etc/logrotate.internal.conf && chown root:root /etc/logrotate.internal.conf && crond -f -l 2"
         depends_on:
             - mariadb
         logging: