diff --git a/.gitignore b/.gitignore index 8fc3c92..8d06017 100644 --- a/.gitignore +++ b/.gitignore @@ -130,5 +130,10 @@ srv/mailman2/ srv/mariadb/ srv/mariadb +# Aether DB Snapshots and Backups +srv/mariadb_bak_* +backups/imported/ +backups/auto_backup_* + srv/aether_api_v5_ln/ srv/aether_api_v5_ln \ No newline at end of file diff --git a/CHEATSHEET.md b/CHEATSHEET.md index 2347a7f..b78ea7f 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -7,8 +7,16 @@ - **API Docs:** [https://dev-api.oneskyit.com/docs](https://dev-api.oneskyit.com/docs) ## 💾 Database Operations -- **Backup:** `./backup_db.sh` -- **Restore:** `./restore_db.sh` (Note: Moves current data to a backup folder first) +- **Manual Backup:** `./backup_db.sh` (Hot backup, live container) +- **Manual Restore:** `./restore_db.sh [path_to_file.gz]` +- **Automated Onsite Import:** + 1. Drop a backup into `backups/import/`. + 2. Run `./check_and_import.sh`. + 3. The file will be restored and moved to `backups/imported/`. + +## ⏰ Scheduling +To backup every hour at 55 minutes past: +`55 * * * * /home/scott/OSIT_dev/aether_container_env/backup_db.sh` ## 📈 Scaling the API 1. Edit `.env` -> `AE_API_REPLICAS=X` diff --git a/aether_fastapi_gunicorn.Dockerfile b/aether_fastapi_gunicorn.Dockerfile index e369b42..34a6828 100644 --- a/aether_fastapi_gunicorn.Dockerfile +++ b/aether_fastapi_gunicorn.Dockerfile @@ -7,7 +7,7 @@ WORKDIR /srv/aether_api RUN apt-get update; \ apt-get install -y \ - imagemagick ffmpeg \ + imagemagick ffmpeg curl \ ; \ rm -rf /var/lib/apt/lists/*; diff --git a/backup_db.sh b/backup_db.sh index 5c89273..b3be2fc 100755 --- a/backup_db.sh +++ b/backup_db.sh @@ -1,7 +1,5 @@ #!/bin/bash # Aether MariaDB Backup Script (Physical Backup) -# Performs a live, hot backup of the running local container. - set -e PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env" @@ -10,14 +8,12 @@ TIMESTAMP=$(date +%Y%m%d_%H%M) BACKUP_FILE="${BACKUP_DIR}/local_backup_${TIMESTAMP}.gz" echo "--- Starting Aether Local Database Backup ---" - -# Ensure backup directory exists mkdir -p "${BACKUP_DIR}" -# Run mariabackup inside the container and stream it to a gzipped file on the host -# We use root here since it's a workstation dev env +# Increased open-files-limit to prevent OS error 24 echo ">>> Backing up to ${BACKUP_FILE}..." -docker exec ae_mariadb_dev mariabackup --user=root --password='$1sky.AE_dev.2023' --backup --stream=xbstream | gzip > "${BACKUP_FILE}" +docker exec ae_mariadb_dev mariabackup --user=root --password='$1sky.AE_dev.2023' \ + --backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}" echo "--- Backup Complete! ---" -ls -lh "${BACKUP_FILE}" +ls -lh "${BACKUP_FILE}" \ No newline at end of file diff --git a/check_and_import.sh b/check_and_import.sh new file mode 100755 index 0000000..a7f5b6b --- /dev/null +++ b/check_and_import.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Aether Automated Import Watchdog +# Checks 'backups/import/' for new database files and restores them. + +set -e + +PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env" +IMPORT_DIR="${PROJECT_ROOT}/backups/import" +ARCHIVE_DIR="${PROJECT_ROOT}/backups/imported" + +mkdir -p "$IMPORT_DIR" "$ARCHIVE_DIR" + +# Find the newest .gz file in the import directory +NEW_BACKUP=$(ls -t "$IMPORT_DIR"/*.gz 2>/dev/null | head -n 1) + +if [ -n "$NEW_BACKUP" ]; then + echo "--- New Backup Detected: $(basename "$NEW_BACKUP") ---" + + # Run the restore + "${PROJECT_ROOT}/restore_db.sh" "$NEW_BACKUP" + + # Move to archive + echo ">>> Archiving imported file..." + mv "$NEW_BACKUP" "$ARCHIVE_DIR/" + + echo "--- Automated Import Finished ---" +else + echo "No files found in $IMPORT_DIR. Nothing to do." +fi diff --git a/conf/aether_fastapi_gunicorn_conf.py b/conf/aether_fastapi_gunicorn_conf.py index 7c2e451..29aad7e 100644 --- a/conf/aether_fastapi_gunicorn_conf.py +++ b/conf/aether_fastapi_gunicorn_conf.py @@ -3,44 +3,25 @@ import os # Gunicorn config variables loglevel = os.getenv('AE_LOG_LVL', 'warning') -accesslog = "/logs/gunicorn_access.log" # "-" # stdout -errorlog = "/logs/gunicorn_error.log" # "-" # stderr -# "logfile" does not seem to actually do anything -# logfile = "/logs/gunicorn.log" # "-" # stderr +accesslog = "-" # stdout +errorlog = "-" # stderr +# ... (existing bind/chdir) ... bind = "0.0.0.0:5005" -# bind = "unix:/tmp/gunicorn.sock" - worker_tmp_dir = "/dev/shm" - chdir = "/srv/aether_api" -# home = /path/to/environment wsgi_app = "app.main:app" -# module = "run_server" -# callable = "app" -# plugins = "python" -# default_proc_name = "app.main:app" -# Setting a long timeout since some FastAPI API requests may take a while -timeout = os.getenv('AE_API_GUNICORN_TIMEOUT', 30) # default 30; 1200 is NOT enough; worker process silent then kill and restart -graceful_timeout = os.getenv('AE_API_GUNICORN_GRACEFUL_TIMEOUT', 30) # default 30; timeout after restart signal; tried 10 2023-07-11 -keepalive = os.getenv('AE_API_GUNICORN_KEEPALIVE', 4) # default 2; not used with sync workers; setting higher because behind load balancer (nginx); tried 10 2023-07-11 +# Numeric variables must be integers +timeout = int(os.getenv('AE_API_GUNICORN_TIMEOUT', 30)) +graceful_timeout = int(os.getenv('AE_API_GUNICORN_GRACEFUL_TIMEOUT', 30)) +keepalive = int(os.getenv('AE_API_GUNICORN_KEEPALIVE', 4)) -# Reload does not work correctly with UvicornWorker -# https://github.com/benoitc/gunicorn/issues/2339 -# Disable reload if using more than one thread reload = False +worker_class = "uvicorn.workers.UvicornWorker" -# reload_engine = "poll" +workers = int(os.getenv('AE_API_GUNICORN_WORKERS', 2)) +threads = int(os.getenv('AE_API_GUNICORN_THREADS', 2)) -worker_class = "uvicorn.workers.UvicornWorker" # default "sync" -# worker_class = "gthread" -# worker_class = "aiohttp.worker.GunicornWebWorker" -# worker_class = "gevent" # for gthread? -# Works are processes, not threads -# workers = 9 # default 1; use 10ish for production; 2 to 4 times the number of cores -# threads = 1 # default 1; only affects Gthread worker type -workers = os.getenv('AE_API_GUNICORN_WORKERS', 2) -threads = os.getenv('AE_API_GUNICORN_THREADS', 2) # umask = '007' diff --git a/conf/crontab b/conf/crontab new file mode 100644 index 0000000..fcc70bd --- /dev/null +++ b/conf/crontab @@ -0,0 +1 @@ +55 * * * * bash /scripts/backup_internal.sh >> /logs/backup_cron.log 2>&1 diff --git a/logs/ae_api/.gitignore b/logs/ae_api/.gitignore old mode 100644 new mode 100755 diff --git a/logs/ae_api_v5/.gitignore b/logs/ae_api_v5/.gitignore old mode 100644 new mode 100755 diff --git a/logs/ae_app/.gitignore b/logs/ae_app/.gitignore old mode 100644 new mode 100755 diff --git a/logs/php7/.gitignore b/logs/php7/.gitignore old mode 100644 new mode 100755 diff --git a/logs/web/nginx/.gitignore b/logs/web/nginx/.gitignore old mode 100644 new mode 100755 diff --git a/restore_db.sh b/restore_db.sh index 439124a..f23672c 100755 --- a/restore_db.sh +++ b/restore_db.sh @@ -1,49 +1,65 @@ #!/bin/bash # Aether MariaDB Restore Script (Physical Backup) -# Automates: Stop -> Backup existing -> Extract -> Prepare -> Fix Perms -> Start - set -e PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env" -BACKUP_FILE="${PROJECT_ROOT}/backups/mariadbbackup_1555.gz" +DEFAULT_BACKUP="${PROJECT_ROOT}/backups/mariadbbackup_1555.gz" +BACKUP_FILE="${1:-$DEFAULT_BACKUP}" + MARIADB_DATA="${PROJECT_ROOT}/srv/mariadb" RESTORE_TEMP="${PROJECT_ROOT}/srv/restore_temp" TIMESTAMP=$(date +%Y%m%d_%H%M%S) +# Load env for password +source "${PROJECT_ROOT}/.env" + +if [ ! -f "$BACKUP_FILE" ]; then + echo "ERROR: Backup file not found: $BACKUP_FILE" + exit 1 +fi + echo "--- Starting Aether Database Restore ---" # 1. Stop MariaDB -echo ">>> Stopping MariaDB container..." +echo ">>> Stopping MariaDB..." cd "${PROJECT_ROOT}" && docker compose stop mariadb # 2. Archive current data -if [ "$(ls -A ${MARIADB_DATA})" ]; then - echo ">>> Archiving current data to srv/mariadb_bak_${TIMESTAMP}..." +if [ -d "$MARIADB_DATA" ] && [ "$(ls -A $MARIADB_DATA)" ]; then + echo ">>> Archiving current data..." mv "${MARIADB_DATA}" "${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}" fi mkdir -p "${MARIADB_DATA}" "${RESTORE_TEMP}" -# 3. Extract and Prepare using Docker -echo ">>> Running extraction and preparation in temporary container..." +# 3. Extract and Prepare +echo ">>> Running extraction and preparation..." docker run --rm --user 0 \ - -v "${PROJECT_ROOT}/backups":/backups \ + -v "${BACKUP_FILE}":/backups/import.gz \ -v "${RESTORE_TEMP}":/restore \ -v "${PROJECT_ROOT}/scripts/restore_internal.sh":/restore.sh \ - mariadb:10.11 bash /restore.sh + mariadb:10.11 bash -c "export BACKUP_FILE=/backups/import.gz && bash /restore.sh" -# 4. Move prepared data to final location -echo ">>> Moving prepared data to srv/mariadb..." -mv "${RESTORE_TEMP}"/* "${MARIADB_DATA}/" -mv "${RESTORE_TEMP}"/.* "${MARIADB_DATA}/" 2>/dev/null || true +# 4. Move prepared data (Using container to avoid permission issues) +echo ">>> Moving prepared data..." +docker run --rm --user 0 \ + -v "${RESTORE_TEMP}":/src \ + -v "${MARIADB_DATA}":/dst \ + alpine sh -c "mv /src/* /dst/ 2>/dev/null || true; mv /src/.* /dst/ 2>/dev/null || true" rmdir "${RESTORE_TEMP}" # 5. Fix Permissions -echo ">>> Fixing ownership for MariaDB user (999:999)..." +echo ">>> Fixing ownership (999:999)..." docker run --rm -v "${MARIADB_DATA}":/var/lib/mysql alpine chown -R 999:999 /var/lib/mysql -# 6. Start MariaDB -echo ">>> Starting MariaDB container..." +# 6. Start MariaDB in Maintenance Mode to reset password +echo ">>> Resetting root password to match local .env..." +docker run -d --name ae_mariadb_maint -v "${MARIADB_DATA}":/var/lib/mysql mariadb:10.11 --skip-grant-tables +sleep 5 +docker exec ae_mariadb_maint mariadb -e "FLUSH PRIVILEGES; ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;" +docker stop ae_mariadb_maint && docker rm ae_mariadb_maint + +# 7. Start MariaDB Normally +echo ">>> Starting MariaDB container normally..." docker compose start mariadb -echo "--- Restore Complete! Check logs with 'docker logs ae_mariadb_dev' ---" - +echo "--- Restore and Password Reset Complete! ---" \ No newline at end of file diff --git a/scripts/backup_internal.sh b/scripts/backup_internal.sh new file mode 100644 index 0000000..2f75232 --- /dev/null +++ b/scripts/backup_internal.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Aether Internal Backup Script (Runs inside the Cron Container) +set -e + +# These are paths INSIDE the cron container +BACKUP_DIR="/backups" +TIMESTAMP=$(date +%Y%m%d_%H%M) +BACKUP_FILE="${BACKUP_DIR}/auto_backup_${TIMESTAMP}.gz" + +echo "[$(date)] Starting Scheduled Backup..." + +# We use the Docker CLI inside this container to talk to the MariaDB container +# The password is taken from the environment variable passed to this service +docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_PASSWORD}" \ + --backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}" + +echo "[$(date)] Backup Complete: ${BACKUP_FILE}" + +# Optional: Clean up backups older than 7 days +find "${BACKUP_DIR}" -name "auto_backup_*.gz" -mtime +7 -delete diff --git a/scripts/restore_internal.sh b/scripts/restore_internal.sh new file mode 100644 index 0000000..65e6545 --- /dev/null +++ b/scripts/restore_internal.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +# Configuration +BACKUP_FILE="${BACKUP_FILE:-/backups/import.gz}" +RESTORE_DIR="/restore" + +echo ">>> Phase 0: Wiping restore directory..." +rm -rf "${RESTORE_DIR:?}"/* + +echo ">>> Phase 1: Extracting ${BACKUP_FILE} to ${RESTORE_DIR}..." +gunzip -c "${BACKUP_FILE}" | mbstream -x -C "${RESTORE_DIR}" + +echo ">>> Phase 2: Fixing checkpoint metadata names..." +cd "${RESTORE_DIR}" +ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints || true +ln -sf mariadb_backup_info xtrabackup_info || true + +echo ">>> Phase 3: Decompressing data..." +mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535 + +echo ">>> Phase 4: Preparing backup (Applying logs)..." +mariabackup --prepare --target-dir="${RESTORE_DIR}" --open-files-limit=65535 + +echo ">>> Restore preparation complete!"