High Availability Patterns

# MySQL Primary-Replica Configuration
[mysqld]
server-id = 1
log_bin = /var/log/mysql/mysql-bin.log
binlog_do_db = production
binlog_format = ROW
max_binlog_size = 100M
expire_logs_days = 7

# Replica configuration
[mysqld]
server-id = 2
relay-log = /var/log/mysql/mysql-relay-bin.log
read_only = 1
super_read_only = 1

# Advanced health check with custom logic
location /health {
  access_log off;
  
  content_by_lua_block {
    local redis = require "resty.redis"
    local mysql = require "resty.mysql"
    
    -- Check Redis connectivity
    local red = redis:new()
    local ok, err = red:connect("127.0.0.1", 6379)
    if not ok then
      ngx.status = 503
      ngx.say("Redis unhealthy: ", err)
      return
    end
    
    -- Check MySQL connectivity
    local db = mysql:new()
    local ok, err = db:connect{
      host = "127.0.0.1",
      port = 3306,
      database = "production"
    }
    if not ok then
      ngx.status = 503
      ngx.say("MySQL unhealthy: ", err)
      return
    end
    
    -- All checks passed
    ngx.status = 200
    ngx.say("Healthy")
  }
}

# Automated recovery script example
#!/bin/bash

MAX_RETRIES=3
RETRY_DELAY=30

recover_service() {
  local service=$1
  local retries=0
  
  while [ $retries -lt $MAX_RETRIES ]; do
    echo "Attempting to recover $service (attempt $((retries+1)))"
    
    # Stop the service
    systemctl stop $service
    sleep 5
    
    # Clear any locks or temp files
    rm -f /var/lock/$service/*
    
    # Start the service
    if systemctl start $service; then
      echo "$service recovered successfully"
      
      # Verify service is healthy
      sleep 10
      if systemctl is-active $service; then
        return 0
      fi
    fi
    
    retries=$((retries+1))
    sleep $RETRY_DELAY
  done
  
  echo "Failed to recover $service after $MAX_RETRIES attempts"
  # Trigger escalation
  notify_oncall "Service $service failed to recover"
  return 1
}

# Example Prometheus alerting rules
groups:
  - name: high_availability
    rules:
    - alert: InstanceDown
      expr: up == 0
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "Instance {{ $labels.instance }} down"
        
    - alert: HighErrorRate
      expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "High error rate detected"
        
    - alert: DatabaseReplicationLag
      expr: mysql_slave_lag_seconds > 10
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Database replication lag exceeds 10 seconds"