Add production database sync script with sanitization (#258)
* Add prod-to-dev database sync script Adds scripts/sync-prod-db.sh and npm run db:sync-prod to safely copy production data to the development database. After restoring, the script nulls all Discord webhook URLs and anonymizes non-admin user PII (email, name, avatar) to prevent notification leaks and reduce PII exposure on developer machines. Admin accounts are preserved intact. https://claude.ai/code/session_01EiYh5ZiuWAnpo1PND45Yi1 * Fix six code review issues in sync-prod-db.sh - Add upfront check for pg_dump/pg_restore/psql availability (#5) - Fix password masking to use sed instead of broken bash glob (#6) - Replace DROP/CREATE DATABASE with pg_restore --clean --if-exists, which works on managed cloud databases without superuser access (#2) - Remove dead ADMIN_DB assignment that was immediately overwritten (#3) - Handle pg_restore exit code 1 (non-fatal warnings) gracefully instead of aborting via set -e (#1) - Count webhook URLs before the UPDATE for an accurate cleared count (#4) https://claude.ai/code/session_01EiYh5ZiuWAnpo1PND45Yi1 --------- Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Chris Parsons
GitHub
parent
1d3eb4aac2
commit
139061e702
2 changed files with 167 additions and 0 deletions
|
|
@ -8,6 +8,7 @@
|
|||
"build:server": "node scripts/build-server.mjs",
|
||||
"db:generate": "dotenv -- drizzle-kit generate",
|
||||
"db:migrate": "dotenv -- drizzle-kit migrate",
|
||||
"db:sync-prod": "bash scripts/sync-prod-db.sh",
|
||||
"dev": "NODE_OPTIONS='--import ./instrument.server.mjs' dotenv -- tsx watch server.ts",
|
||||
"start": "NODE_ENV=production NODE_OPTIONS='--import ./instrument.server.mjs' node dist/server.js",
|
||||
"start:production": "NODE_ENV=production NODE_OPTIONS='--import ./instrument.server.mjs' node dist/server.js",
|
||||
|
|
|
|||
166
scripts/sync-prod-db.sh
Executable file
166
scripts/sync-prod-db.sh
Executable file
|
|
@ -0,0 +1,166 @@
|
|||
#!/usr/bin/env bash
|
||||
# sync-prod-db.sh
|
||||
#
|
||||
# Dumps the production database and restores it to the development database,
|
||||
# then sanitizes the copy to prevent outbound notifications and reduce PII exposure.
|
||||
#
|
||||
# Usage:
|
||||
# PROD_DATABASE_URL="postgres://..." npm run db:sync-prod
|
||||
# # or pass as argument:
|
||||
# bash scripts/sync-prod-db.sh "postgres://..."
|
||||
#
|
||||
# Requirements: pg_dump, pg_restore, psql (PostgreSQL client tools)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Colour helpers ────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
GREEN='\033[0;32m'
|
||||
BOLD='\033[1m'
|
||||
RESET='\033[0m'
|
||||
|
||||
info() { echo -e "${BOLD}$*${RESET}"; }
|
||||
warn() { echo -e "${YELLOW}$*${RESET}"; }
|
||||
success() { echo -e "${GREEN}$*${RESET}"; }
|
||||
error() { echo -e "${RED}ERROR: $*${RESET}" >&2; exit 1; }
|
||||
|
||||
# ── Check required tools (fix #5) ────────────────────────────────────────────
|
||||
for cmd in pg_dump pg_restore psql; do
|
||||
command -v "$cmd" &>/dev/null || error "'$cmd' not found. Install PostgreSQL client tools (e.g. brew install libpq)."
|
||||
done
|
||||
|
||||
# ── Load .env ─────────────────────────────────────────────────────────────────
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ENV_FILE="$SCRIPT_DIR/../.env"
|
||||
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
# Export only DATABASE_URL from .env (avoid polluting env with unrelated vars)
|
||||
DATABASE_URL_FROM_ENV=$(grep -E '^DATABASE_URL=' "$ENV_FILE" | head -1 | cut -d= -f2- | sed 's/^"//' | sed 's/"$//')
|
||||
if [[ -n "$DATABASE_URL_FROM_ENV" && -z "${DATABASE_URL:-}" ]]; then
|
||||
export DATABASE_URL="$DATABASE_URL_FROM_ENV"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Resolve URLs ──────────────────────────────────────────────────────────────
|
||||
# PROD_DATABASE_URL: env var or first positional argument
|
||||
PROD_DB="${PROD_DATABASE_URL:-${1:-}}"
|
||||
DEV_DB="${DATABASE_URL:-}"
|
||||
|
||||
[[ -z "$PROD_DB" ]] && error "PROD_DATABASE_URL is not set.\nSet it as an environment variable or pass it as the first argument:\n PROD_DATABASE_URL=\"postgres://...\" npm run db:sync-prod"
|
||||
[[ -z "$DEV_DB" ]] && error "DATABASE_URL is not set. Add it to your .env file."
|
||||
|
||||
# Guard against accidentally syncing prod → prod
|
||||
if [[ "$PROD_DB" == "$DEV_DB" ]]; then
|
||||
error "PROD_DATABASE_URL and DATABASE_URL are the same. Aborting to prevent data loss."
|
||||
fi
|
||||
|
||||
# ── Mask password for display (fix #6) ───────────────────────────────────────
|
||||
# Replaces only the password portion (between : and @) using sed, not bash globs.
|
||||
mask_url() {
|
||||
echo "$1" | sed 's|\(://[^:]*\):[^@]*@|\1:***@|'
|
||||
}
|
||||
|
||||
# ── Confirmation prompt ───────────────────────────────────────────────────────
|
||||
echo ""
|
||||
warn "⚠️ WARNING: This will OVERWRITE your development database with production data."
|
||||
echo ""
|
||||
echo -e " Source (prod): ${BOLD}$(mask_url "$PROD_DB")${RESET}"
|
||||
echo -e " Target (dev): ${BOLD}$(mask_url "$DEV_DB")${RESET}"
|
||||
echo ""
|
||||
warn "After restore, the script will:"
|
||||
echo " • NULL out all Discord webhook URLs (prevents notification leaks)"
|
||||
echo " • Anonymize email/name/avatar for non-admin users (reduces PII exposure)"
|
||||
echo " • Admin accounts are left untouched so you can still access admin pages"
|
||||
echo ""
|
||||
read -r -p "Type 'yes' to continue: " CONFIRM
|
||||
if [[ "$CONFIRM" != "yes" ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Dump ──────────────────────────────────────────────────────────────────────
|
||||
DUMP_FILE=$(mktemp /tmp/brackt_prod_dump_XXXXXX.dump)
|
||||
# Ensure the temp file is cleaned up on exit
|
||||
trap 'rm -f "$DUMP_FILE"' EXIT
|
||||
|
||||
info ""
|
||||
info "1/3 Dumping production database…"
|
||||
pg_dump \
|
||||
--no-owner \
|
||||
--no-acl \
|
||||
--format=custom \
|
||||
"$PROD_DB" \
|
||||
--file="$DUMP_FILE"
|
||||
success " Dump complete: $(du -h "$DUMP_FILE" | cut -f1)"
|
||||
|
||||
# ── Restore (fix #1, #2, #3) ─────────────────────────────────────────────────
|
||||
# Uses --clean --if-exists to drop and recreate all objects within the existing
|
||||
# database, without needing DROP DATABASE / CREATE DATABASE permissions.
|
||||
# This works on managed cloud databases (Railway, Render, Neon, Supabase, etc.).
|
||||
#
|
||||
# pg_restore exits with code 1 for non-fatal warnings (e.g. "role X does not
|
||||
# exist" with --no-owner), so we capture the exit code and only abort on
|
||||
# unexpected failures, not routine restore warnings.
|
||||
info ""
|
||||
info "2/3 Restoring production dump to development database…"
|
||||
RESTORE_EXIT=0
|
||||
pg_restore \
|
||||
--no-owner \
|
||||
--no-acl \
|
||||
--clean \
|
||||
--if-exists \
|
||||
--dbname="$DEV_DB" \
|
||||
"$DUMP_FILE" || RESTORE_EXIT=$?
|
||||
|
||||
if [[ $RESTORE_EXIT -eq 0 ]]; then
|
||||
success " Restore complete."
|
||||
elif [[ $RESTORE_EXIT -eq 1 ]]; then
|
||||
warn " Restore completed with warnings (non-fatal). Continuing…"
|
||||
else
|
||||
error "pg_restore failed with exit code $RESTORE_EXIT. Aborting."
|
||||
fi
|
||||
|
||||
# ── Sanitize (fix #4) ────────────────────────────────────────────────────────
|
||||
info ""
|
||||
info "3/3 Sanitizing development database…"
|
||||
|
||||
SANITIZE_RESULT=$(psql "$DEV_DB" --tuples-only --no-align << 'SQL'
|
||||
BEGIN;
|
||||
|
||||
-- 1. Count leagues with webhooks before clearing (accurate count)
|
||||
SELECT 'discord_webhooks_nulled:' || COUNT(*) FROM leagues WHERE discord_webhook_url IS NOT NULL;
|
||||
UPDATE leagues SET discord_webhook_url = NULL;
|
||||
|
||||
-- 2. Anonymize PII for non-admin users only
|
||||
-- (admin accounts are untouched so admin pages remain accessible)
|
||||
UPDATE users SET
|
||||
email = 'user_' || id || '@example.com',
|
||||
first_name = 'Test',
|
||||
last_name = 'User',
|
||||
display_name = 'Test User',
|
||||
image_url = NULL
|
||||
WHERE is_admin = FALSE;
|
||||
SELECT 'users_anonymized:' || COUNT(*) FROM users WHERE is_admin = FALSE;
|
||||
|
||||
SELECT 'admin_users_kept:' || COUNT(*) FROM users WHERE is_admin = TRUE;
|
||||
|
||||
COMMIT;
|
||||
SQL
|
||||
)
|
||||
|
||||
LEAGUES_COUNT=$(echo "$SANITIZE_RESULT" | grep 'discord_webhooks_nulled:' | cut -d: -f2 | tr -d ' ')
|
||||
ANON_COUNT=$(echo "$SANITIZE_RESULT" | grep 'users_anonymized:' | cut -d: -f2 | tr -d ' ')
|
||||
ADMIN_COUNT=$(echo "$SANITIZE_RESULT" | grep 'admin_users_kept:' | cut -d: -f2 | tr -d ' ')
|
||||
|
||||
echo ""
|
||||
success "✓ Sync complete. Sanitization summary:"
|
||||
echo " • Discord webhook URLs cleared: ${LEAGUES_COUNT:-?} leagues"
|
||||
echo " • Non-admin users anonymized: ${ANON_COUNT:-?} users"
|
||||
echo " • Admin accounts preserved: ${ADMIN_COUNT:-?} users"
|
||||
echo ""
|
||||
warn "Reminder: ensure your .env does NOT contain:"
|
||||
echo " • RESEND_API_KEY (email sending)"
|
||||
echo " • SENTRY_AUTH_TOKEN (error tracking)"
|
||||
echo " • Production Clerk keys (use dev Clerk instance)"
|
||||
echo ""
|
||||
Loading…
Add table
Reference in a new issue