Enhance DB automation: Added conference export, absolute path restore logic, and automated multi-user credential resetting. Updated README and CHEATSHEET.
This commit is contained in:
Scott Idem
@@ -8,7 +8,8 @@
|
||||
|
||||
## 💾 Database Operations
|
||||
- **Manual Backup:** `./backup_db.sh` (Hot backup, live container)
|
||||
- **Manual Restore:** `./restore_db.sh [path_to_file.gz]`
|
||||
- **Manual Restore:** `./restore_db.sh [path_to_file.gz]` (Automated password/grant reset)
|
||||
- **Conference Export:** `./export_db.sh` (Saves to `backups/conference_export/`)
|
||||
- **Automated Onsite Import:**
|
||||
1. Drop a backup into `backups/import/`.
|
||||
2. Run `./check_and_import.sh`.
|
||||
|
||||
36
README.md
36
README.md
@@ -214,3 +214,39 @@ ln -s /mnt/data_drive/srv/data/osit_app/hosted_files_dev /home/scott/OSIT_dev/ae
|
||||
ln -s /mnt/data_drive/srv/data/osit_app/hosted_tmp /home/scott/OSIT_dev/aether_container_env/srv/hosted_tmp_link
|
||||
ln -s /mnt/data_drive/srv/data/osit_app/hosted_tmp_dev /home/scott/OSIT_dev/aether_container_env/srv/hosted_tmp_dev_link
|
||||
|
||||
## Database Management (Physical Backups & Restores)
|
||||
|
||||
The system uses physical hot backups via `mariabackup` for speed and data integrity.
|
||||
|
||||
### User-Facing Scripts
|
||||
These scripts are located in the `aether_container_env/` root directory.
|
||||
|
||||
* **`./backup_db.sh`**:
|
||||
* Triggers an immediate hot backup of the local MariaDB instance.
|
||||
* Stores results in `backups/local_backup_YYYYMMDD_HHMM.gz`.
|
||||
* **`./export_db.sh`**:
|
||||
* Creates a "Conference Ready" backup.
|
||||
* Triggers a backup inside the `ae_ops_dev` container and moves it to `backups/conference_export/`.
|
||||
* Ensures the resulting file is owned by the host user (UID 1000).
|
||||
* **`./restore_db.sh [backup_file.gz]`**:
|
||||
* Performs a full "Clean Slate" restoration from a `.gz` stream file.
|
||||
* **Logic**: Stops MariaDB -> Archives current data (`srv/mariadb_bak_...`) -> Wipes `srv/mariadb` -> Extracts/Prepares data -> Resets `root` and `aether_dev` passwords to match your current `.env`.
|
||||
* **`./check_and_import.sh`**:
|
||||
* Watchdog script that monitors `backups/import/` for new `.gz` files.
|
||||
* If a file is found, it triggers `./restore_db.sh` and then moves the processed file to `backups/imported/`.
|
||||
|
||||
### Internal Logic Scripts
|
||||
These are located in `aether_container_env/scripts/` and are primarily called by the user-facing scripts or the `ae_ops` cron service.
|
||||
|
||||
* **`scripts/backup_internal.sh`**:
|
||||
* Runs inside the `ae_ops` container.
|
||||
* Connects to MariaDB, streams the backup, and fixes file ownership for the host.
|
||||
* **`scripts/restore_internal.sh`**:
|
||||
* Runs inside a temporary restoration container.
|
||||
* Handles decompression, metadata linking (handling `mariadb_backup_...` vs legacy `xtrabackup_...` filenames), and log application (`--prepare`).
|
||||
|
||||
### Backup Directory Structure
|
||||
* `backups/`: General local backups.
|
||||
* `backups/import/`: Drop files here for `check_and_import.sh`.
|
||||
* `backups/imported/`: Archive of processed import files.
|
||||
* `backups/conference_export/`: Specialized manual exports.
|
||||
35
export_db.sh
Executable file
35
export_db.sh
Executable file
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# Aether Conference Export Script
|
||||
# Manually triggers a hot backup for off-site use.
|
||||
|
||||
set -e
|
||||
|
||||
PROJECT_ROOT="/home/scott/OSIT_dev/aether_container_env"
|
||||
EXPORT_DIR="${PROJECT_ROOT}/backups/conference_export"
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
EXPORT_FILE="conference_backup_${TIMESTAMP}.gz"
|
||||
|
||||
mkdir -p "$EXPORT_DIR"
|
||||
|
||||
echo "--- Starting Conference Database Export ---"
|
||||
|
||||
# Trigger the internal backup script inside the ops container
|
||||
# This will create an 'auto_backup_...' file in the backups folder
|
||||
docker exec ae_ops_dev bash /scripts/backup_internal.sh
|
||||
|
||||
# Find the most recent backup file created in the backups folder
|
||||
LATEST_BACKUP=$(ls -t "${PROJECT_ROOT}/backups"/auto_backup_*.gz | head -n 1)
|
||||
|
||||
if [ -n "$LATEST_BACKUP" ]; then
|
||||
echo ">>> Moving latest backup to export directory: ${EXPORT_FILE}"
|
||||
mv "$LATEST_BACKUP" "${EXPORT_DIR}/${EXPORT_FILE}"
|
||||
|
||||
# Ensure final ownership is correct
|
||||
chown 1000:1000 "${EXPORT_DIR}/${EXPORT_FILE}"
|
||||
|
||||
echo "--- Export Complete! ---"
|
||||
echo "File location: ${EXPORT_DIR}/${EXPORT_FILE}"
|
||||
else
|
||||
echo "ERROR: Failed to find the generated backup file."
|
||||
exit 1
|
||||
fi
|
||||
@@ -18,6 +18,9 @@ if [ ! -f "$BACKUP_FILE" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Convert to absolute path for Docker volume mounting
|
||||
BACKUP_FILE_ABS=$(readlink -f "$BACKUP_FILE")
|
||||
|
||||
echo "--- Starting Aether Database Restore ---"
|
||||
|
||||
# 1. Stop MariaDB
|
||||
@@ -27,14 +30,17 @@ cd "${PROJECT_ROOT}" && docker compose stop mariadb
|
||||
# 2. Archive current data
|
||||
if [ -d "$MARIADB_DATA" ] && [ "$(ls -A $MARIADB_DATA)" ]; then
|
||||
echo ">>> Archiving current data..."
|
||||
mv "${MARIADB_DATA}" "${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}"
|
||||
BACKUP_DIR="${PROJECT_ROOT}/srv/mariadb_bak_${TIMESTAMP}"
|
||||
mv "${MARIADB_DATA}" "${BACKUP_DIR}"
|
||||
# Fix ownership of archived data so host user can manage it
|
||||
docker run --rm -v "${BACKUP_DIR}":/bak alpine chown -R 1000:1000 /bak
|
||||
fi
|
||||
mkdir -p "${MARIADB_DATA}" "${RESTORE_TEMP}"
|
||||
|
||||
# 3. Extract and Prepare
|
||||
echo ">>> Running extraction and preparation..."
|
||||
docker run --rm --user 0 \
|
||||
-v "${BACKUP_FILE}":/backups/import.gz \
|
||||
-v "${BACKUP_FILE_ABS}":/backups/import.gz \
|
||||
-v "${RESTORE_TEMP}":/restore \
|
||||
-v "${PROJECT_ROOT}/scripts/restore_internal.sh":/restore.sh \
|
||||
mariadb:10.11 bash -c "export BACKUP_FILE=/backups/import.gz && bash /restore.sh"
|
||||
@@ -52,10 +58,19 @@ echo ">>> Fixing ownership (999:999)..."
|
||||
docker run --rm -v "${MARIADB_DATA}":/var/lib/mysql alpine chown -R 999:999 /var/lib/mysql
|
||||
|
||||
# 6. Start MariaDB in Maintenance Mode to reset password
|
||||
echo ">>> Resetting root password to match local .env..."
|
||||
echo ">>> Resetting passwords to match local .env..."
|
||||
docker run -d --name ae_mariadb_maint -v "${MARIADB_DATA}":/var/lib/mysql mariadb:10.11 --skip-grant-tables
|
||||
sleep 5
|
||||
docker exec ae_mariadb_maint mariadb -e "FLUSH PRIVILEGES; ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_PASSWORD}'; GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}' WITH GRANT OPTION; FLUSH PRIVILEGES;"
|
||||
# Maintenance SQL: Sets root password AND ensures app user exists with correct password/grants
|
||||
MAINT_SQL="FLUSH PRIVILEGES;
|
||||
ALTER USER 'root'@'localhost' IDENTIFIED BY '${AE_DB_ROOT_PASSWORD}';
|
||||
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '${AE_DB_ROOT_PASSWORD}' WITH GRANT OPTION;
|
||||
CREATE USER IF NOT EXISTS '${AE_DB_USERNAME}'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}';
|
||||
ALTER USER '${AE_DB_USERNAME}'@'%' IDENTIFIED BY '${AE_DB_PASSWORD}';
|
||||
GRANT ALL PRIVILEGES ON \`${AE_DB_NAME}\`.* TO '${AE_DB_USERNAME}'@'%';
|
||||
FLUSH PRIVILEGES;"
|
||||
|
||||
docker exec ae_mariadb_maint mariadb -e "$MAINT_SQL"
|
||||
docker stop ae_mariadb_maint && docker rm ae_mariadb_maint
|
||||
|
||||
# 7. Start MariaDB Normally
|
||||
|
||||
@@ -11,10 +11,13 @@ echo "[$(date)] Starting Scheduled Backup..."
|
||||
|
||||
# We use the Docker CLI inside this container to talk to the MariaDB container
|
||||
# The password is taken from the environment variable passed to this service
|
||||
docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_PASSWORD}" \
|
||||
docker exec ${CONTAINER_MARIADB} mariabackup --user=root --password="${AE_DB_ROOT_PASSWORD}" \
|
||||
--backup --stream=xbstream --open-files-limit=65535 | gzip > "${BACKUP_FILE}"
|
||||
|
||||
echo "[$(date)] Backup Complete: ${BACKUP_FILE}"
|
||||
|
||||
# Ensure host user can manage the backup files
|
||||
chown 1000:1000 "${BACKUP_FILE}"
|
||||
|
||||
# Optional: Clean up backups older than 7 days
|
||||
find "${BACKUP_DIR}" -name "auto_backup_*.gz" -mtime +7 -delete
|
||||
|
||||
@@ -11,10 +11,16 @@ rm -rf "${RESTORE_DIR:?}"/*
|
||||
echo ">>> Phase 1: Extracting ${BACKUP_FILE} to ${RESTORE_DIR}..."
|
||||
gunzip -c "${BACKUP_FILE}" | mbstream -x -C "${RESTORE_DIR}"
|
||||
|
||||
echo ">>> Phase 2: Fixing checkpoint metadata names..."
|
||||
echo ">>> Phase 2: Metadata Check..."
|
||||
cd "${RESTORE_DIR}"
|
||||
ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints || true
|
||||
ln -sf mariadb_backup_info xtrabackup_info || true
|
||||
if [ -f "mariadb_backup_checkpoints" ] && [ ! -f "xtrabackup_checkpoints" ]; then
|
||||
echo ">>> Linking mariadb_backup_checkpoints to xtrabackup_checkpoints..."
|
||||
ln -sf mariadb_backup_checkpoints xtrabackup_checkpoints
|
||||
fi
|
||||
if [ -f "mariadb_backup_info" ] && [ ! -f "xtrabackup_info" ]; then
|
||||
echo ">>> Linking mariadb_backup_info to xtrabackup_info..."
|
||||
ln -sf mariadb_backup_info xtrabackup_info
|
||||
fi
|
||||
|
||||
echo ">>> Phase 3: Decompressing data..."
|
||||
mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535
|
||||
@@ -22,4 +28,4 @@ mariabackup --decompress --target-dir="${RESTORE_DIR}" --open-files-limit=65535
|
||||
echo ">>> Phase 4: Preparing backup (Applying logs)..."
|
||||
mariabackup --prepare --target-dir="${RESTORE_DIR}" --open-files-limit=65535
|
||||
|
||||
echo ">>> Restore preparation complete!"
|
||||
echo ">>> Restore preparation complete!"
|
||||
Reference in New Issue
Block a user