diff --git a/CHEATSHEET.md b/CHEATSHEET.md index b78ea7f..f16ddc9 100644 --- a/CHEATSHEET.md +++ b/CHEATSHEET.md @@ -8,7 +8,8 @@ ## 💾 Database Operations - **Manual Backup:** `./backup_db.sh` (Hot backup, live container) -- **Manual Restore:** `./restore_db.sh [path_to_file.gz]` +- **Manual Restore:** `./restore_db.sh [path_to_file.gz]` (Automated password/grant reset) +- **Conference Export:** `./export_db.sh` (Saves to `backups/conference_export/`) - **Automated Onsite Import:** 1. Drop a backup into `backups/import/`. 2. Run `./check_and_import.sh`. diff --git a/README.md b/README.md index 59164b2..958630c 100644 --- a/README.md +++ b/README.md @@ -214,3 +214,39 @@ ln -s /mnt/data_drive/srv/data/osit_app/hosted_files_dev /home/scott/OSIT_dev/ae ln -s /mnt/data_drive/srv/data/osit_app/hosted_tmp /home/scott/OSIT_dev/aether_container_env/srv/hosted_tmp_link ln -s /mnt/data_drive/srv/data/osit_app/hosted_tmp_dev /home/scott/OSIT_dev/aether_container_env/srv/hosted_tmp_dev_link +## Database Management (Physical Backups & Restores) + +The system uses physical hot backups via `mariabackup` for speed and data integrity. + +### User-Facing Scripts +These scripts are located in the `aether_container_env/` root directory. + +* **`./backup_db.sh`**: + * Triggers an immediate hot backup of the local MariaDB instance. + * Stores results in `backups/local_backup_YYYYMMDD_HHMM.gz`. +* **`./export_db.sh`**: + * Creates a "Conference Ready" backup. + * Triggers a backup inside the `ae_ops_dev` container and moves it to `backups/conference_export/`. + * Ensures the resulting file is owned by the host user (UID 1000). +* **`./restore_db.sh [backup_file.gz]`**: + * Performs a full "Clean Slate" restoration from a `.gz` stream file. + * **Logic**: Stops MariaDB -> Archives current data (`srv/mariadb_bak_...`) -> Wipes `srv/mariadb` -> Extracts/Prepares data -> Resets `root` and `aether_dev` passwords to match your current `.env`. +* **`./check_and_import.sh`**: + * Watchdog script that monitors `backups/import/` for new `.gz` files. + * If a file is found, it triggers `./restore_db.sh` and then moves the processed file to `backups/imported/`. + +### Internal Logic Scripts +These are located in `aether_container_env/scripts/` and are primarily called by the user-facing scripts or the `ae_ops` cron service. + +* **`scripts/backup_internal.sh`**: + * Runs inside the `ae_ops` container. + * Connects to MariaDB, streams the backup, and fixes file ownership for the host. +* **`scripts/restore_internal.sh`**: + * Runs inside a temporary restoration container. + * Handles decompression, metadata linking (handling `mariadb_backup_...` vs legacy `xtrabackup_...` filenames), and log application (`--prepare`). + +### Backup Directory Structure +* `backups/`: General local backups. +* `backups/import/`: Drop files here for `check_and_import.sh`. +* `backups/imported/`: Archive of processed import files. +* `backups/conference_export/`: Specialized manual exports. \ No newline at end of file diff --git a/export_db.sh b/export_db.sh new file mode 100755 index 0000000..a4139f5 --- /dev/null +++ b/export_db.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Aether Conference Export Script +# Manually triggers a hot backup for off-site use. + +set -e + +PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env" +EXPORT_DIR="${PROJECT_ROOT}/backups/conference_export" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +EXPORT_FILE="conference_backup_${TIMESTAMP}.gz" + +mkdir -p "$EXPORT_DIR" + +echo "--- Starting Conference Database Export ---" + +# Trigger the internal backup script inside the ops container +# This will create an 'auto_backup_...' file in the backups folder +docker exec ae_ops_dev bash /scripts/backup_internal.sh + +# Find the most recent backup file created in the backups folder +LATEST_BACKUP=$(ls -t "${PROJECT_ROOT}/backups"/auto_backup_*.gz | head -n 1) + +if [ -n "$LATEST_BACKUP" ]; then + echo ">>> Moving latest backup to export directory: ${EXPORT_FILE}" + mv "$LATEST_BACKUP" "${EXPORT_DIR}/${EXPORT_FILE}" + + # Ensure final ownership is correct + chown 1000:1000 "${EXPORT_DIR}/${EXPORT_FILE}" + + echo "--- Export Complete! ---" + echo "File location: ${EXPORT_DIR}/${EXPORT_FILE}" +else + echo "ERROR: Failed to find the generated backup file." + exit 1 +fi diff --git a/restore_db.sh b/restore_db.sh index f23672c..dbe57a3 100755 --- a/restore_db.sh +++ b/restore_db.sh @@ -18,6 +18,9 @@ if [ ! -f "$BACKUP_FILE" ]; then exit 1 fi +# Convert to absolute path for Docker volume mounting +BACKUP_FILE_ABS=$(readlink -f "$BACKUP_FILE") + echo "--- Starting Aether Database Restore ---" # 1. Stop MariaDB @@ -27,14 +30,17 @@ cd "${PROJECT_ROOT}" && docker compose stop mariadb # 2. Archive current data if [ -d "$MARIADB_DATA" ] && [ "$(ls -A $MARIADB_DATA)" ]; then echo ">>> Archiving current data..." - mv "${MARIADB_DATA}" "${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}" + BACKUP_DIR="${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}" + mv "${MARIADB_DATA}" "${BACKUP_DIR}" + # Fix ownership of archived data so host user can manage it + docker run --rm -v "${BACKUP_DIR}":/bak alpine chown -R 1000:1000 /bak fi mkdir -p "${MARIADB_DATA}" "${RESTORE_TEMP}" # 3. Extract and Prepare echo ">>> Running extraction and preparation..." docker run --rm --user 0 \ - -v "${BACKUP_FILE}":/backups/import.gz \ + -v "${BACKUP_FILE_ABS}":/backups/import.gz \ -v "${RESTORE_TEMP}":/restore \ -v "${PROJECT_ROOT}/scripts/restore_internal.sh":/restore.sh \ mariadb:10.11 bash -c "export BACKUP_FILE=/backups/import.gz && bash /restore.sh" @@ -52,10 +58,19 @@ echo ">>> Fixing ownership (999:999)..." docker run --rm -v "${MARIADB_DATA}":/var/lib/mysql alpine chown -R 999:999 /var/lib/mysql # 6. Start MariaDB in Maintenance Mode to reset password -echo ">>> Resetting root password to match local .env..." +echo ">>> Resetting passwords to match local .env..." docker run -d --name ae_mariadb_maint -v "${MARIADB_DATA}":/var/lib/mysql mariadb:10.11 --skip-grant-tables sleep 5 -docker exec ae_mariadb_maint mariadb -e "FLUSH PRIVILEGES; ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;" +# Maintenance SQL: Sets root password AND ensures app user exists with correct password/grants +MAINT_SQL="FLUSH PRIVILEGES; +ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_ROOT_PASSWORD}'; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_ROOT_PASSWORD}' WITH GRANT OPTION; +CREATE USER IF NOT EXISTS '${AE_DB_USERNAME}'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}'; +ALTER USER '${AE_DB_USERNAME}'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}'; +GRANT ALL PRIVILEGES ON \`${AE_DB_NAME}\`.* TO '${AE_DB_USERNAME}'@'%'; +FLUSH PRIVILEGES;" + +docker exec ae_mariadb_maint mariadb -e "$MAINT_SQL" docker stop ae_mariadb_maint && docker rm ae_mariadb_maint # 7. Start MariaDB Normally diff --git a/scripts/backup_internal.sh b/scripts/backup_internal.sh index 2f75232..d494528 100644 --- a/scripts/backup_internal.sh +++ b/scripts/backup_internal.sh @@ -11,10 +11,13 @@ echo "[$(date)] Starting Scheduled Backup..." # We use the Docker CLI inside this container to talk to the MariaDB container # The password is taken from the environment variable passed to this service -docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_PASSWORD}" \ +docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_ROOT_PASSWORD}" \ --backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}" echo "[$(date)] Backup Complete: ${BACKUP_FILE}" +# Ensure host user can manage the backup files +chown 1000:1000 "${BACKUP_FILE}" + # Optional: Clean up backups older than 7 days find "${BACKUP_DIR}" -name "auto_backup_*.gz" -mtime +7 -delete diff --git a/scripts/restore_internal.sh b/scripts/restore_internal.sh index 65e6545..9a78440 100644 --- a/scripts/restore_internal.sh +++ b/scripts/restore_internal.sh @@ -11,10 +11,16 @@ rm -rf "${RESTORE_DIR:?}"/* echo ">>> Phase 1: Extracting ${BACKUP_FILE} to ${RESTORE_DIR}..." gunzip -c "${BACKUP_FILE}" | mbstream -x -C "${RESTORE_DIR}" -echo ">>> Phase 2: Fixing checkpoint metadata names..." +echo ">>> Phase 2: Metadata Check..." cd "${RESTORE_DIR}" -ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints || true -ln -sf mariadb_backup_info xtrabackup_info || true +if [ -f "mariadb_backup_checkpoints" ] && [ ! -f "xtrabackup_checkpoints" ]; then + echo ">>> Linking mariadb_backup_checkpoints to xtrabackup_checkpoints..." + ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints +fi +if [ -f "mariadb_backup_info" ] && [ ! -f "xtrabackup_info" ]; then + echo ">>> Linking mariadb_backup_info to xtrabackup_info..." + ln -sf mariadb_backup_info xtrabackup_info +fi echo ">>> Phase 3: Decompressing data..." mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535 @@ -22,4 +28,4 @@ mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535 echo ">>> Phase 4: Preparing backup (Applying logs)..." mariabackup --prepare --target-dir="${RESTORE_DIR}" --open-files-limit=65535 -echo ">>> Restore preparation complete!" +echo ">>> Restore preparation complete!" \ No newline at end of file