Zero-downtime blue/green deploy via nginx

Tokopedia gak boleh down 1 menit pun di jam ramai. Blue/green deploy: jalankan 2 instance aplikasi (biru lagi serve, hijau standby), deploy versi baru ke hijau, smoke test, lalu switch nginx ke hijau. Kalau ada error, swap balik. Snippet ini script orchestrator dengan health check.

Kode

#!/usr/bin/env bash
# /opt/deploy/blue_green.sh
# Usage: ./blue_green.sh <new-image-tag>

set -euo pipefail

readonly APP_NAME="tokopedia-api"
readonly NEW_TAG="${1:?Pakai: $0 <image-tag>}"
readonly NGINX_UPSTREAM="/etc/nginx/upstream.conf"
readonly HEALTH_PATH="/healthz"
readonly WARMUP_REQUESTS=50
readonly HEALTH_TIMEOUT=120

# Port assignment
declare -A PORTS=([blue]=8081 [green]=8082)

log() {
  echo "[$(date -u +%H:%M:%S)] $*"
}

# Cek slot aktif sekarang berdasarkan upstream config
current_slot() {
  if grep -q "server 127.0.0.1:${PORTS[blue]}" "${NGINX_UPSTREAM}" 2>/dev/null; then
    echo "blue"
  else
    echo "green"
  fi
}

other_slot() {
  if [[ "$1" == "blue" ]]; then echo "green"; else echo "blue"; fi
}

deploy_to_slot() {
  local slot=$1
  local port=${PORTS[$slot]}
  local container="${APP_NAME}-${slot}"

  log "Stop container lama (kalau ada): ${container}"
  docker rm -f "${container}" 2>/dev/null || true

  log "Pull image: ${APP_NAME}:${NEW_TAG}"
  docker pull "${APP_NAME}:${NEW_TAG}"

  log "Start container ${container} di port ${port}"
  docker run -d \
    --name "${container}" \
    --restart unless-stopped \
    -p "127.0.0.1:${port}:8080" \
    --env-file /etc/${APP_NAME}/env \
    "${APP_NAME}:${NEW_TAG}"
}

wait_healthy() {
  local port=$1
  local deadline=$(( $(date +%s) + HEALTH_TIMEOUT ))

  log "Tunggu health check di port ${port}"
  while (( $(date +%s) < deadline )); do
    if curl -sf -o /dev/null -w "%{http_code}" \
        "http://127.0.0.1:${port}${HEALTH_PATH}" | grep -q "^200$"; then
      log "Health check OK di port ${port}"
      return 0
    fi
    sleep 2
  done

  log "ERROR: health check timeout di port ${port}"
  return 1
}

warmup() {
  local port=$1
  log "Warmup ${WARMUP_REQUESTS} request ke port ${port}"

  for i in $(seq 1 "${WARMUP_REQUESTS}"); do
    curl -sf -o /dev/null "http://127.0.0.1:${port}${HEALTH_PATH}" || true
  done
  log "Warmup selesai"
}

switch_nginx() {
  local target_slot=$1
  local target_port=${PORTS[$target_slot]}

  log "Switch nginx upstream ke ${target_slot} (port ${target_port})"

  cat > "${NGINX_UPSTREAM}" <<EOF
upstream app_backend {
  server 127.0.0.1:${target_port};
  keepalive 32;
}
EOF

  if ! nginx -t; then
    log "ERROR: nginx config invalid"
    return 1
  fi

  nginx -s reload
  log "Nginx reloaded — traffic sekarang ke ${target_slot}"
}

main() {
  local active
  active=$(current_slot)
  local standby
  standby=$(other_slot "${active}")

  log "Active: ${active}, deploy ke standby: ${standby}"

  deploy_to_slot "${standby}"

  if ! wait_healthy "${PORTS[$standby]}"; then
    log "Deploy gagal, container standby di-stop"
    docker rm -f "${APP_NAME}-${standby}"
    exit 1
  fi

  warmup "${PORTS[$standby]}"
  switch_nginx "${standby}"

  log "Grace period 30s sebelum stop slot lama"
  sleep 30

  log "Stop container slot lama: ${active}"
  docker rm -f "${APP_NAME}-${active}"

  log "=== Deploy ${NEW_TAG} selesai ke ${standby} ==="
}

main "$@"

#!/usr/bin/env bash
# /opt/deploy/rollback.sh — swap balik dalam 5 detik

set -euo pipefail

readonly NGINX_UPSTREAM="/etc/nginx/upstream.conf"

active=$(grep -oE "8081|8082" "${NGINX_UPSTREAM}" | head -1)

if [[ "${active}" == "8081" ]]; then
  rollback_port=8082
  rollback_name="green"
else
  rollback_port=8081
  rollback_name="blue"
fi

echo "Rollback ke ${rollback_name} (port ${rollback_port})"

if ! curl -sf -o /dev/null "http://127.0.0.1:${rollback_port}/healthz"; then
  echo "ERROR: target rollback tidak healthy. Manual intervention dibutuhkan."
  exit 1
fi

cat > "${NGINX_UPSTREAM}" <<EOF
upstream app_backend {
  server 127.0.0.1:${rollback_port};
  keepalive 32;
}
EOF

nginx -t && nginx -s reload
echo "Rolled back."

Pemakaian

# Deploy versi baru
sudo /opt/deploy/blue_green.sh v2.34.1

# Kalau ada error setelah switch, rollback
sudo /opt/deploy/rollback.sh

# Cek status
docker ps | grep tokopedia-api
# tokopedia-api-blue   v2.34.0   Up 2 days
# tokopedia-api-green  v2.34.1   Up 5 minutes

# /etc/nginx/sites-enabled/api.conf
upstream app_backend {
  server 127.0.0.1:8081;
  keepalive 32;
}

server {
  listen 443 ssl http2;
  server_name api.tokopedia-clone.id;

  location / {
    proxy_pass http://app_backend;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_set_header Host $host;
    proxy_read_timeout 60s;
  }
}

Kapan dipakai

• Production API yang butuh SLA tinggi.
• Aplikasi yang punya schema migration backward compatible (tidak in-place breaking).
• Service yang JIT-warmup-sensitive (JVM, .NET).
• Stateful service yang gak boleh restart in-place (cache memory).

Catatan

• DB migration harus backward compatible — slot lama masih bisa baca/tulis schema baru. Pakai expand-migrate-contract pattern.
• Session sticky — kalau pakai in-memory session, switch bikin user logout. Pakai Redis / JWT supaya stateless.
• Grace period sebelum stop slot lama — biar request inflight selesai. 30-60 detik biasanya cukup.
• Health check beda dari readiness — health = “alive?”, readiness = “siap terima traffic?”. Warmup pengaruhi readiness, bukan health.
• Resource 2x — slot ganda butuh memory dan CPU 2x sementara. Plan capacity.
• Database connection pool — slot baru bikin pool baru. Total connection ke DB sementara 2x. Cek max_connections Postgres.

Untuk Kubernetes, pakai Service + Deployment label switch (atau pakai Argo Rollouts / Flagger). Pattern sama, cuma orchestrator-nya k8s.