Automation: Final robust DB restore fixes and updated gitignore rules.

.gitignore

@@ -130,5 +130,10 @@ srv/mailman2/
 srv/mariadb/
 srv/mariadb
 
+# Aether DB Snapshots and Backups
+srv/mariadb_bak_*
+backups/imported/
+backups/auto_backup_*
+
 srv/aether_api_v5_ln/
 srv/aether_api_v5_ln

CHEATSHEET.md

@@ -7,8 +7,16 @@
 - **API Docs:** [https://dev-api.oneskyit.com/docs](https://dev-api.oneskyit.com/docs)
 
 ## 💾 Database Operations
-- **Backup:** `./backup_db.sh`
-- **Restore:** `./restore_db.sh` (Note: Moves current data to a backup folder first)
+- **Manual Backup:** `./backup_db.sh` (Hot backup, live container)
+- **Manual Restore:** `./restore_db.sh [path_to_file.gz]`
+- **Automated Onsite Import:**
+    1. Drop a backup into `backups/import/`.
+    2. Run `./check_and_import.sh`.
+    3. The file will be restored and moved to `backups/imported/`.
+
+## ⏰ Scheduling
+To backup every hour at 55 minutes past:
+`55 * * * * /home/scott/OSIT_dev/aether_container_env/backup_db.sh`
 
 ## 📈 Scaling the API
 1. Edit `.env` -> `AE_API_REPLICAS=X`

aether_fastapi_gunicorn.Dockerfile

@@ -7,7 +7,7 @@ WORKDIR /srv/aether_api
 
 RUN apt-get update; \
     apt-get install -y \
-    imagemagick ffmpeg \
+    imagemagick ffmpeg curl \
     ; \
     rm -rf /var/lib/apt/lists/*;
 

backup_db.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 # Aether MariaDB Backup Script (Physical Backup)
-# Performs a live, hot backup of the running local container.
-
 set -e
 
 PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env"
@@ -10,14 +8,12 @@ TIMESTAMP=$(date +%Y%m%d_%H%M)
 BACKUP_FILE="${BACKUP_DIR}/local_backup_${TIMESTAMP}.gz"
 
 echo "--- Starting Aether Local Database Backup ---"
-
-# Ensure backup directory exists
 mkdir -p "${BACKUP_DIR}"
 
-# Run mariabackup inside the container and stream it to a gzipped file on the host
-# We use root here since it's a workstation dev env
+# Increased open-files-limit to prevent OS error 24
 echo ">>> Backing up to ${BACKUP_FILE}..."
-docker exec ae_mariadb_dev mariabackup --user=root --password='$1sky.AE_dev.2023' --backup --stream=xbstream | gzip > "${BACKUP_FILE}"
+docker exec ae_mariadb_dev mariabackup --user=root --password='$1sky.AE_dev.2023' \
+    --backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}"
 
 echo "--- Backup Complete! ---"
-ls -lh "${BACKUP_FILE}"
+ls -lh "${BACKUP_FILE}"

check_and_import.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Aether Automated Import Watchdog
+# Checks 'backups/import/' for new database files and restores them.
+
+set -e
+
+PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env"
+IMPORT_DIR="${PROJECT_ROOT}/backups/import"
+ARCHIVE_DIR="${PROJECT_ROOT}/backups/imported"
+
+mkdir -p "$IMPORT_DIR" "$ARCHIVE_DIR"
+
+# Find the newest .gz file in the import directory
+NEW_BACKUP=$(ls -t "$IMPORT_DIR"/*.gz 2>/dev/null | head -n 1)
+
+if [ -n "$NEW_BACKUP" ]; then
+    echo "--- New Backup Detected: $(basename "$NEW_BACKUP") ---"
+    
+    # Run the restore
+    "${PROJECT_ROOT}/restore_db.sh" "$NEW_BACKUP"
+    
+    # Move to archive
+    echo ">>> Archiving imported file..."
+    mv "$NEW_BACKUP" "$ARCHIVE_DIR/"
+    
+    echo "--- Automated Import Finished ---"
+else
+    echo "No files found in $IMPORT_DIR. Nothing to do."
+fi

conf/aether_fastapi_gunicorn_conf.py

@@ -3,44 +3,25 @@ import os
 # Gunicorn config variables
 loglevel = os.getenv('AE_LOG_LVL', 'warning')
 
-accesslog = "/logs/gunicorn_access.log" # "-"  # stdout
-errorlog = "/logs/gunicorn_error.log" # "-"  # stderr
-# "logfile" does not seem to actually do anything
-# logfile = "/logs/gunicorn.log" # "-"  # stderr
+accesslog = "-"  # stdout
+errorlog = "-"  # stderr
 
+# ... (existing bind/chdir) ...
 bind = "0.0.0.0:5005"
-# bind = "unix:/tmp/gunicorn.sock"
-
 worker_tmp_dir = "/dev/shm"
-
 chdir = "/srv/aether_api"
-# home = /path/to/environment
 wsgi_app = "app.main:app"
-# module = "run_server"
-# callable = "app"
-# plugins = "python"
-# default_proc_name = "app.main:app"
 
-# Setting a long timeout since some FastAPI API requests may take a while
-timeout = os.getenv('AE_API_GUNICORN_TIMEOUT', 30) # default 30; 1200 is NOT enough; worker process silent then kill and restart
-graceful_timeout = os.getenv('AE_API_GUNICORN_GRACEFUL_TIMEOUT', 30) # default 30; timeout after restart signal; tried 10 2023-07-11
-keepalive = os.getenv('AE_API_GUNICORN_KEEPALIVE', 4) # default 2; not used with sync workers; setting higher because behind load balancer (nginx); tried 10 2023-07-11
+# Numeric variables must be integers
+timeout = int(os.getenv('AE_API_GUNICORN_TIMEOUT', 30))
+graceful_timeout = int(os.getenv('AE_API_GUNICORN_GRACEFUL_TIMEOUT', 30))
+keepalive = int(os.getenv('AE_API_GUNICORN_KEEPALIVE', 4))
 
-# Reload does not work correctly with UvicornWorker
-# https://github.com/benoitc/gunicorn/issues/2339
-# Disable reload if using more than one thread
 reload = False
+worker_class = "uvicorn.workers.UvicornWorker"
 
-# reload_engine = "poll"
+workers = int(os.getenv('AE_API_GUNICORN_WORKERS', 2))
+threads = int(os.getenv('AE_API_GUNICORN_THREADS', 2))
 
-worker_class = "uvicorn.workers.UvicornWorker" # default "sync"
-# worker_class = "gthread"
-# worker_class = "aiohttp.worker.GunicornWebWorker"
-# worker_class = "gevent" # for gthread?
-# Works are processes, not threads
-# workers = 9 # default 1; use 10ish for production; 2 to 4 times the number of cores
-# threads = 1 # default 1; only affects Gthread worker type
-workers = os.getenv('AE_API_GUNICORN_WORKERS', 2)
-threads = os.getenv('AE_API_GUNICORN_THREADS', 2)
 
 # umask = '007'

conf/crontab

@@ -0,0 +1 @@
+55 * * * * bash /scripts/backup_internal.sh >> /logs/backup_cron.log 2>&1

restore_db.sh

@@ -1,49 +1,65 @@
 #!/bin/bash
 # Aether MariaDB Restore Script (Physical Backup)
-# Automates: Stop -> Backup existing -> Extract -> Prepare -> Fix Perms -> Start
-
 set -e
 
 PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env"
-BACKUP_FILE="${PROJECT_ROOT}/backups/mariadbbackup_1555.gz"
+DEFAULT_BACKUP="${PROJECT_ROOT}/backups/mariadbbackup_1555.gz"
+BACKUP_FILE="${1:-$DEFAULT_BACKUP}"
+
 MARIADB_DATA="${PROJECT_ROOT}/srv/mariadb"
 RESTORE_TEMP="${PROJECT_ROOT}/srv/restore_temp"
 TIMESTAMP=$(date +%Y%m%d_%H%M%S)
 
+# Load env for password
+source "${PROJECT_ROOT}/.env"
+
+if [ ! -f "$BACKUP_FILE" ]; then
+    echo "ERROR: Backup file not found: $BACKUP_FILE"
+    exit 1
+fi
+
 echo "--- Starting Aether Database Restore ---"
 
 # 1. Stop MariaDB
-echo ">>> Stopping MariaDB container..."
+echo ">>> Stopping MariaDB..."
 cd "${PROJECT_ROOT}" && docker compose stop mariadb
 
 # 2. Archive current data
-if [ "$(ls -A ${MARIADB_DATA})" ]; then
-    echo ">>> Archiving current data to srv/mariadb_bak_${TIMESTAMP}..."
+if [ -d "$MARIADB_DATA" ] && [ "$(ls -A $MARIADB_DATA)" ]; then
+    echo ">>> Archiving current data..."
     mv "${MARIADB_DATA}" "${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}"
 fi
 mkdir -p "${MARIADB_DATA}" "${RESTORE_TEMP}"
 
-# 3. Extract and Prepare using Docker
-echo ">>> Running extraction and preparation in temporary container..."
+# 3. Extract and Prepare
+echo ">>> Running extraction and preparation..."
 docker run --rm --user 0 \
-    -v "${PROJECT_ROOT}/backups":/backups \
+    -v "${BACKUP_FILE}":/backups/import.gz \
     -v "${RESTORE_TEMP}":/restore \
     -v "${PROJECT_ROOT}/scripts/restore_internal.sh":/restore.sh \
-    mariadb:10.11 bash /restore.sh
+    mariadb:10.11 bash -c "export BACKUP_FILE=/backups/import.gz && bash /restore.sh"
 
-# 4. Move prepared data to final location
-echo ">>> Moving prepared data to srv/mariadb..."
-mv "${RESTORE_TEMP}"/* "${MARIADB_DATA}/"
-mv "${RESTORE_TEMP}"/.* "${MARIADB_DATA}/" 2>/dev/null || true
+# 4. Move prepared data (Using container to avoid permission issues)
+echo ">>> Moving prepared data..."
+docker run --rm --user 0 \
+    -v "${RESTORE_TEMP}":/src \
+    -v "${MARIADB_DATA}":/dst \
+    alpine sh -c "mv /src/* /dst/ 2>/dev/null || true; mv /src/.* /dst/ 2>/dev/null || true"
 rmdir "${RESTORE_TEMP}"
 
 # 5. Fix Permissions
-echo ">>> Fixing ownership for MariaDB user (999:999)..."
+echo ">>> Fixing ownership (999:999)..."
 docker run --rm -v "${MARIADB_DATA}":/var/lib/mysql alpine chown -R 999:999 /var/lib/mysql
 
-# 6. Start MariaDB
-echo ">>> Starting MariaDB container..."
+# 6. Start MariaDB in Maintenance Mode to reset password
+echo ">>> Resetting root password to match local .env..."
+docker run -d --name ae_mariadb_maint -v "${MARIADB_DATA}":/var/lib/mysql mariadb:10.11 --skip-grant-tables
+sleep 5
+docker exec ae_mariadb_maint mariadb -e "FLUSH PRIVILEGES; ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;"
+docker stop ae_mariadb_maint && docker rm ae_mariadb_maint
+
+# 7. Start MariaDB Normally
+echo ">>> Starting MariaDB container normally..."
 docker compose start mariadb
 
-echo "--- Restore Complete! Check logs with 'docker logs ae_mariadb_dev' ---"
-
+echo "--- Restore and Password Reset Complete! ---"

scripts/backup_internal.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Aether Internal Backup Script (Runs inside the Cron Container)
+set -e
+
+# These are paths INSIDE the cron container
+BACKUP_DIR="/backups"
+TIMESTAMP=$(date +%Y%m%d_%H%M)
+BACKUP_FILE="${BACKUP_DIR}/auto_backup_${TIMESTAMP}.gz"
+
+echo "[$(date)] Starting Scheduled Backup..."
+
+# We use the Docker CLI inside this container to talk to the MariaDB container
+# The password is taken from the environment variable passed to this service
+docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_PASSWORD}" \
+    --backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}"
+
+echo "[$(date)] Backup Complete: ${BACKUP_FILE}"
+
+# Optional: Clean up backups older than 7 days
+find "${BACKUP_DIR}" -name "auto_backup_*.gz" -mtime +7 -delete

scripts/restore_internal.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+# Configuration
+BACKUP_FILE="${BACKUP_FILE:-/backups/import.gz}"
+RESTORE_DIR="/restore"
+
+echo ">>> Phase 0: Wiping restore directory..."
+rm -rf "${RESTORE_DIR:?}"/*
+
+echo ">>> Phase 1: Extracting ${BACKUP_FILE} to ${RESTORE_DIR}..."
+gunzip -c "${BACKUP_FILE}" | mbstream -x -C "${RESTORE_DIR}"
+
+echo ">>> Phase 2: Fixing checkpoint metadata names..."
+cd "${RESTORE_DIR}"
+ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints || true
+ln -sf mariadb_backup_info xtrabackup_info || true
+
+echo ">>> Phase 3: Decompressing data..."
+mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535
+
+echo ">>> Phase 4: Preparing backup (Applying logs)..."
+mariabackup --prepare --target-dir="${RESTORE_DIR}" --open-files-limit=65535
+
+echo ">>> Restore preparation complete!"