#!/bin/bash ############################################################################### # mockupAWS Database Backup Script v1.0.0 # # Description: Automated PostgreSQL backup with encryption and S3 upload # # Features: # - Daily full backups (pg_dump) # - Continuous WAL archiving # - AES-256 encryption # - S3/GCS upload with multi-region replication # - Backup integrity verification # - 30-day retention policy # # Usage: # ./scripts/backup.sh full # Full backup # ./scripts/backup.sh wal # WAL archive # ./scripts/backup.sh verify # Verify backup integrity # ./scripts/backup.sh cleanup # Clean old backups # # Environment Variables: # DATABASE_URL - PostgreSQL connection string (required) # BACKUP_BUCKET - S3 bucket name (required) # BACKUP_REGION - AWS region (default: us-east-1) # BACKUP_ENCRYPTION_KEY - AES-256 encryption key (required) # BACKUP_RETENTION_DAYS - Retention period (default: 30) # AWS_ACCESS_KEY_ID - AWS credentials # AWS_SECRET_ACCESS_KEY - AWS credentials # ############################################################################### set -euo pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" BACKUP_DIR="${PROJECT_ROOT}/storage/backups" LOG_DIR="${PROJECT_ROOT}/storage/logs" TIMESTAMP=$(date +%Y%m%d_%H%M%S) DATE=$(date +%Y%m%d) # Default values BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-30} BACKUP_REGION=${BACKUP_REGION:-us-east-1} BACKUP_BUCKET=${BACKUP_BUCKET:-} BACKUP_SECONDARY_REGION=${BACKUP_SECONDARY_REGION:-eu-west-1} BACKUP_SECONDARY_BUCKET=${BACKUP_SECONDARY_BUCKET:-} # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Logging log() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" } log_success() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] ✓${NC} $1" } log_warn() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] ⚠${NC} $1" } log_error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ✗${NC} $1" } # Create directories mkdir -p "$BACKUP_DIR" "$LOG_DIR" # Validate environment validate_env() { local missing=() if [[ -z "${DATABASE_URL:-}" ]]; then missing+=("DATABASE_URL") fi if [[ -z "${BACKUP_BUCKET:-}" ]]; then log_warn "BACKUP_BUCKET not set - backups will be stored locally only" fi if [[ -z "${BACKUP_ENCRYPTION_KEY:-}" ]]; then log_warn "BACKUP_ENCRYPTION_KEY not set - backups will not be encrypted" fi if [[ ${#missing[@]} -gt 0 ]]; then log_error "Missing required environment variables: ${missing[*]}" exit 1 fi } # Extract connection details from DATABASE_URL parse_database_url() { local url="$1" # Remove protocol local conn="${url#postgresql://}" conn="${conn#postgresql+asyncpg://}" conn="${conn#postgres://}" # Parse user:password@host:port/database if [[ "$conn" =~ ^([^:]+):([^@]+)@([^:]+):?([0-9]*)/([^?]+) ]]; then DB_USER="${BASH_REMATCH[1]}" DB_PASS="${BASH_REMATCH[2]}" DB_HOST="${BASH_REMATCH[3]}" DB_PORT="${BASH_REMATCH[4]:-5432}" DB_NAME="${BASH_REMATCH[5]}" else log_error "Could not parse DATABASE_URL" exit 1 fi export PGPASSWORD="$DB_PASS" } # Encrypt file encrypt_file() { local input_file="$1" local output_file="$2" if [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]]; then openssl enc -aes-256-cbc -salt -pbkdf2 \ -in "$input_file" \ -out "$output_file" \ -pass pass:"$BACKUP_ENCRYPTION_KEY" 2>/dev/null log "File encrypted: $output_file" else cp "$input_file" "$output_file" log_warn "No encryption key - file copied without encryption" fi } # Decrypt file decrypt_file() { local input_file="$1" local output_file="$2" if [[ -n "${BACKUP_ENCRYPTION_KEY:-}" ]]; then openssl enc -aes-256-cbc -d -pbkdf2 \ -in "$input_file" \ -out "$output_file" \ -pass pass:"$BACKUP_ENCRYPTION_KEY" 2>/dev/null log "File decrypted: $output_file" else cp "$input_file" "$output_file" fi } # Calculate checksum calculate_checksum() { local file="$1" sha256sum "$file" | awk '{print $1}' } # Upload to S3 upload_to_s3() { local file="$1" local key="$2" local bucket="${3:-$BACKUP_BUCKET}" local region="${4:-$BACKUP_REGION}" if [[ -z "$bucket" ]]; then log_warn "S3 bucket not configured - skipping upload" return 0 fi log "Uploading to S3: s3://$bucket/$key" aws s3 cp "$file" "s3://$bucket/$key" \ --region "$region" \ --storage-class STANDARD_IA \ --metadata "backup-date=$TIMESTAMP,checksum=$(calculate_checksum "$file")" log_success "Uploaded to S3: s3://$bucket/$key" } # Upload to secondary region (DR) upload_to_secondary() { local file="$1" local key="$2" if [[ -n "${BACKUP_SECONDARY_BUCKET:-}" ]]; then log "Replicating to secondary region: $BACKUP_SECONDARY_REGION" upload_to_s3 "$file" "$key" "$BACKUP_SECONDARY_BUCKET" "$BACKUP_SECONDARY_REGION" fi } # Full database backup backup_full() { log "Starting full database backup..." parse_database_url "$DATABASE_URL" local backup_name="mockupaws_full_${TIMESTAMP}" local backup_file="${BACKUP_DIR}/${backup_name}.sql" local compressed_file="${backup_file}.gz" local encrypted_file="${compressed_file}.enc" local checksum_file="${backup_file}.sha256" local s3_key="backups/full/${DATE}/${backup_name}.sql.gz.enc" # Create backup log "Dumping database: $DB_NAME" pg_dump \ --host="$DB_HOST" \ --port="$DB_PORT" \ --username="$DB_USER" \ --dbname="$DB_NAME" \ --format=custom \ --compress=9 \ --verbose \ --file="$backup_file" \ 2>"${LOG_DIR}/backup_${TIMESTAMP}.log" # Compress log "Compressing backup..." gzip -f "$backup_file" # Encrypt log "Encrypting backup..." encrypt_file "$compressed_file" "$encrypted_file" rm -f "$compressed_file" # Calculate checksum local checksum checksum=$(calculate_checksum "$encrypted_file") echo "$checksum $(basename "$encrypted_file")" > "$checksum_file" # Upload to S3 upload_to_s3 "$encrypted_file" "$s3_key" upload_to_secondary "$encrypted_file" "$s3_key" upload_to_s3 "$checksum_file" "${s3_key}.sha256" # Create metadata file cat > "${backup_file}.json" </dev/null || stat -c%s "$encrypted_file"), "retention_days": $BACKUP_RETENTION_DAYS, "s3_location": "s3://$BACKUP_BUCKET/$s3_key" } EOF upload_to_s3 "${backup_file}.json" "${s3_key}.json" # Cleanup local files (keep last 3) log "Cleaning up local backup files..." ls -t "${BACKUP_DIR}"/mockupaws_full_*.sql.gz.enc 2>/dev/null | tail -n +4 | xargs -r rm -f log_success "Full backup completed: $backup_name" echo "Backup location: s3://$BACKUP_BUCKET/$s3_key" # Record in database record_backup "full" "$s3_key" "$checksum" } # WAL archive backup backup_wal() { log "Starting WAL archive backup..." parse_database_url "$DATABASE_URL" local wal_dir="${BACKUP_DIR}/wal" mkdir -p "$wal_dir" # Trigger WAL switch psql \ --host="$DB_HOST" \ --port="$DB_PORT" \ --username="$DB_USER" \ --dbname="$DB_NAME" \ --command="SELECT pg_switch_wal();" \ --tuples-only \ --no-align \ 2>/dev/null || true # Archive WAL files local wal_files=() for wal_file in "$wal_dir"/*.backup 2>/dev/null; do if [[ -f "$wal_file" ]]; then wal_files+=("$wal_file") fi done if [[ ${#wal_files[@]} -eq 0 ]]; then log_warn "No WAL files to archive" return 0 fi local archive_name="wal_${TIMESTAMP}.tar.gz" local archive_path="${BACKUP_DIR}/${archive_name}" local encrypted_archive="${archive_path}.enc" local s3_key="backups/wal/${DATE}/${archive_name}.enc" # Create archive tar -czf "$archive_path" -C "$wal_dir" . # Encrypt encrypt_file "$archive_path" "$encrypted_archive" rm -f "$archive_path" # Upload upload_to_s3 "$encrypted_archive" "$s3_key" upload_to_secondary "$encrypted_archive" "$s3_key" # Cleanup rm -f "$encrypted_archive" rm -f "$wal_dir"/*.backup log_success "WAL archive completed: ${#wal_files[@]} files archived" } # Verify backup integrity verify_backup() { local backup_file="$1" log "Verifying backup: $backup_file" if [[ ! -f "$backup_file" ]]; then log_error "Backup file not found: $backup_file" exit 1 fi # Decrypt local decrypted_file="${backup_file%.enc}" decrypt_file "$backup_file" "$decrypted_file" # Decompress if compressed local sql_file="$decrypted_file" if [[ "$decrypted_file" == *.gz ]]; then sql_file="${decrypted_file%.gz}" gunzip -c "$decrypted_file" > "$sql_file" rm -f "$decrypted_file" fi # Verify PostgreSQL custom format if pg_restore --list "$sql_file" > /dev/null 2>&1; then log_success "Backup verification passed: $backup_file" local object_count object_count=$(pg_restore --list "$sql_file" | wc -l) log " Objects in backup: $object_count" else log_error "Backup verification failed: $backup_file" rm -f "$sql_file" exit 1 fi # Cleanup rm -f "$sql_file" } # Cleanup old backups cleanup_old_backups() { log "Cleaning up backups older than $BACKUP_RETENTION_DAYS days..." local cutoff_date cutoff_date=$(date -d "$BACKUP_RETENTION_DAYS days ago" +%Y%m%d 2>/dev/null || date -v-${BACKUP_RETENTION_DAYS}d +%Y%m%d) if [[ -n "${BACKUP_BUCKET:-}" ]]; then # List and delete old S3 backups log "Checking S3 for old backups..." aws s3 ls "s3://$BACKUP_BUCKET/backups/full/" --recursive | \ while read -r line; do local file_date file_date=$(echo "$line" | awk '{print $1}' | tr -d '-') local file_key file_key=$(echo "$line" | awk '{print $4}') if [[ "$file_date" < "$cutoff_date" ]]; then log "Deleting old backup: $file_key" aws s3 rm "s3://$BACKUP_BUCKET/$file_key" fi done fi # Cleanup local backups find "$BACKUP_DIR" -name "mockupaws_full_*.sql.gz.enc" -mtime +$BACKUP_RETENTION_DAYS -delete find "$BACKUP_DIR" -name "wal_*.tar.gz.enc" -mtime +$BACKUP_RETENTION_DAYS -delete log_success "Cleanup completed" } # Record backup in database record_backup() { local backup_type="$1" local s3_key="$2" local checksum="$3" parse_database_url "$DATABASE_URL" psql \ --host="$DB_HOST" \ --port="$DB_PORT" \ --username="$DB_USER" \ --dbname="$DB_NAME" \ --command=" INSERT INTO backup_history (backup_type, s3_key, checksum, status, created_at) VALUES ('$backup_type', '$s3_key', '$checksum', 'completed', NOW()); " \ 2>/dev/null || log_warn "Could not record backup in database" } # List available backups list_backups() { log "Available backups:" if [[ -n "${BACKUP_BUCKET:-}" ]]; then echo -e "\n${GREEN}S3 Backups:${NC}" aws s3 ls "s3://$BACKUP_BUCKET/backups/full/" --recursive | tail -20 fi echo -e "\n${GREEN}Local Backups:${NC}" ls -lh "$BACKUP_DIR"/*.enc 2>/dev/null | tail -10 || echo "No local backups found" } # Main command handler case "${1:-}" in full) validate_env backup_full ;; wal) validate_env backup_wal ;; verify) if [[ -z "${2:-}" ]]; then log_error "Usage: $0 verify " exit 1 fi verify_backup "$2" ;; cleanup) cleanup_old_backups ;; list) list_backups ;; *) echo "mockupAWS Database Backup Script v1.0.0" echo "" echo "Usage: $0 [options]" echo "" echo "Commands:" echo " full Create a full database backup" echo " wal Archive WAL files" echo " verify Verify backup integrity" echo " cleanup Remove old backups (respects retention policy)" echo " list List available backups" echo "" echo "Environment Variables:" echo " DATABASE_URL - PostgreSQL connection string (required)" echo " BACKUP_BUCKET - S3 bucket name" echo " BACKUP_REGION - AWS region (default: us-east-1)" echo " BACKUP_ENCRYPTION_KEY - AES-256 encryption key" echo " BACKUP_RETENTION_DAYS - Retention period (default: 30)" echo "" exit 1 ;; esac