Keepalived High Availability – : VRRP Complete Setup Guide

# Check keepalived process status
ps aux | grep keepalived
systemctl status keepalived

# Verify VRRP instance configuration
ip addr show | grep -E "inet.*scope global"
ip route show table all | grep vrrp

# Monitor VRRP state transitions
journalctl -u keepalived -f
tail -f /var/log/keepalived.log

# Check virtual IP assignment
ip addr show | grep -A5 -B5 "192.168.100.10"

# Capture VRRP advertisement packets
tcpdump -i eth0 -n proto 112
wireshark -i eth0 -f "ip proto 112"

# Monitor network interface statistics
cat /proc/net/dev | grep eth0
ethtool -S eth0 | grep -i error

# Examine current VRRP instances
cat /proc/net/vrrp
grep -r "vrrp_instance" /etc/keepalived/

# Monitor VRRP advertisements
tcpdump -i eth0 -n -v proto 112 | head -20

# Check current master status
ip addr show | grep -E "192\.168\.100\.[0-9]+" 
cat /proc/net/arp | grep "192.168.100.10"

# Verify backup node readiness
systemctl is-active keepalived
journalctl -u keepalived --since "1 hour ago" | grep -i backup

# Generate authentication password
openssl rand -hex 8
echo "auth_pass $(openssl rand -hex 8)" >> /tmp/vrrp_auth.conf

# Validate authentication configuration
keepalived -t -f /etc/keepalived/keepalived.conf

# Update package repositories
sudo apt update && sudo apt upgrade -y

# Install keepalived and dependencies
sudo apt install keepalived -y
sudo apt install ipvsadm iptables-persistent -y

# Verify installation
keepalived --version
which keepalived
dpkg -l | grep keepalived

# Enable EPEL repository (CentOS/RHEL)
sudo dnf install epel-release -y
# OR for older versions:
# sudo yum install epel-release -y

# Install keepalived package
sudo dnf install keepalived -y
sudo dnf install ipvsadm iptables-services -y

# Start and enable services
sudo systemctl start keepalived
sudo systemctl enable keepalived
sudo systemctl status keepalived

# Refresh repositories
sudo zypper refresh

# Install keepalived components
sudo zypper install keepalived -y
sudo zypper install ipvsadm netcat-openbsd -y

# Configure firewall for VRRP
sudo firewall-cmd --permanent --add-protocol=vrrp
sudo firewall-cmd --reload

# Install build dependencies
sudo apt install build-essential libssl-dev libnl-3-dev libnl-genl-3-dev -y

# Download and compile keepalived
wget https://www.keepalived.org/software/keepalived-2.2.8.tar.gz
tar xzf keepalived-2.2.8.tar.gz && cd keepalived-2.2.8

# Configure and compile
./configure --enable-vrrp --enable-lvs --enable-sha1 --enable-snmp
make && sudo make install

# Create systemd service file
sudo cp /usr/local/etc/sysconfig/keepalived /etc/default/keepalived

# Create configuration directory
sudo mkdir -p /etc/keepalived/scripts
sudo touch /etc/keepalived/keepalived.conf

# Set appropriate permissions
sudo chmod 644 /etc/keepalived/keepalived.conf
sudo chown root:root /etc/keepalived/keepalived.conf

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
    notification_email {
        admin@company.com
        sysadmin@company.com
    }
    notification_email_from keepalived@server01.local
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id LB_DEVEL_01
    vrrp_skip_check_adv_addr
    vrrp_strict
    vrrp_garp_interval 0
    vrrp_gna_interval 0
}

vrrp_script chk_haproxy {
    script "/etc/keepalived/scripts/check_haproxy.sh"
    interval 2
    weight -2
    fall 3
    rise 2
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 110
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass mypassword123
    }
    virtual_ipaddress {
        192.168.100.10/24
    }
    track_script {
        chk_haproxy
    }
    notify_master /etc/keepalived/scripts/master.sh
    notify_backup /etc/keepalived/scripts/backup.sh
    notify_fault /etc/keepalived/scripts/fault.sh
}
EOF

# Test configuration syntax
sudo keepalived -t -f /etc/keepalived/keepalived.conf

# Check for configuration errors
sudo keepalived -P -D -f /etc/keepalived/keepalived.conf
journalctl -u keepalived --since "5 minutes ago" | grep -i error

# Validate VRRP parameters
grep -E "(virtual_router_id|priority|advert_int)" /etc/keepalived/keepalived.conf

# Configure email notifications
cat << 'EOF' | sudo tee -a /etc/keepalived/keepalived.conf
global_defs {
    router_id $(hostname -s)
    script_user keepalived_script
    enable_script_security
    vrrp_version 3
    vrrp_iptables
    vrrp_check_unicast_src
}
EOF

# Create script user for security
sudo useradd -r -s /bin/false keepalived_script
sudo mkdir -p /var/run/keepalived
sudo chown keepalived_script:keepalived_script /var/run/keepalived

# Master server configuration
cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id MASTER_SERVER_01
    script_user keepalived_script
    enable_script_security
}

vrrp_script chk_nginx {
    script "/etc/keepalived/scripts/check_nginx.sh"
    interval 2
    weight -2
    fall 3
    rise 2
    user keepalived_script
}

vrrp_instance VI_1 {
    state MASTER
    interface ens192
    virtual_router_id 51
    priority 110
    advert_int 1
    preempt_delay 5
    authentication {
        auth_type PASS
        auth_pass SecurePass2024
    }
    virtual_ipaddress {
        192.168.100.10/24 dev ens192
        192.168.100.11/24 dev ens192
    }
    virtual_routes {
        0.0.0.0/0 via 192.168.100.1
    }
    track_script {
        chk_nginx
    }
    notify_master "/etc/keepalived/scripts/master.sh"
    notify_backup "/etc/keepalived/scripts/backup.sh"
    notify_fault "/etc/keepalived/scripts/fault.sh"
    notify_stop "/etc/keepalived/scripts/stop.sh"
}
EOF

# Create nginx health check script
cat << 'EOF' | sudo tee /etc/keepalived/scripts/check_nginx.sh
#!/bin/bash

if [ -f /var/run/nginx.pid ]; then
    nginx_pid=$(cat /var/run/nginx.pid)
    if kill -0 "$nginx_pid" 2>/dev/null; then
        # Check if nginx is responding
        if curl -f http://localhost:80/ >/dev/null 2>&1; then
            exit 0
        fi
    fi
fi

# Attempt to start nginx if it's not running
systemctl start nginx >/dev/null 2>&1
sleep 2

# Final check
if curl -f http://localhost:80/ >/dev/null 2>&1; then
    exit 0
else
    exit 1
fi
EOF

# Make script executable
sudo chmod +x /etc/keepalived/scripts/check_nginx.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/check_nginx.sh

# Backup server configuration
cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id BACKUP_SERVER_01
    script_user keepalived_script
    enable_script_security
}

vrrp_script chk_nginx {
    script "/etc/keepalived/scripts/check_nginx.sh"
    interval 2
    weight -2
    fall 3
    rise 2
    user keepalived_script
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens192
    virtual_router_id 51
    priority 100
    advert_int 1
    preempt_delay 5
    authentication {
        auth_type PASS
        auth_pass SecurePass2024
    }
    virtual_ipaddress {
        192.168.100.10/24 dev ens192
        192.168.100.11/24 dev ens192
    }
    virtual_routes {
        0.0.0.0/0 via 192.168.100.1
    }
    track_script {
        chk_nginx
    }
    notify_master "/etc/keepalived/scripts/master.sh"
    notify_backup "/etc/keepalived/scripts/backup.sh"
    notify_fault "/etc/keepalived/scripts/fault.sh"
    notify_stop "/etc/keepalived/scripts/stop.sh"
}
EOF

# Master notification script
cat << 'EOF' | sudo tee /etc/keepalived/scripts/master.sh
#!/bin/bash
logger "Keepalived: Transitioning to MASTER state"
echo "$(date): Became MASTER" >> /var/log/keepalived-state.log

# Enable services when becoming master
systemctl start nginx
systemctl start haproxy

# Update DNS or load balancer configuration if needed
# curl -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$RECORD_ID" \
#   -H "Authorization: Bearer $API_TOKEN" \
#   -H "Content-Type: application/json" \
#   --data '{"type":"A","name":"service.example.com","content":"192.168.100.10"}'
EOF

# Backup notification script
cat << 'EOF' | sudo tee /etc/keepalived/scripts/backup.sh
#!/bin/bash
logger "Keepalived: Transitioning to BACKUP state"
echo "$(date): Became BACKUP" >> /var/log/keepalived-state.log

# Optional: Stop services when becoming backup to save resources
# systemctl stop nginx
# systemctl stop haproxy
EOF

# Fault notification script
cat << 'EOF' | sudo tee /etc/keepalived/scripts/fault.sh
#!/bin/bash
logger "Keepalived: Entering FAULT state"
echo "$(date): Entered FAULT state" >> /var/log/keepalived-state.log

# Send alert notification
echo "Server $(hostname) keepalived FAULT at $(date)" | mail -s "Keepalived FAULT Alert" admin@company.com
EOF

# Set permissions for all scripts
sudo chmod +x /etc/keepalived/scripts/*.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/*.sh

# Start keepalived on both nodes
sudo systemctl start keepalived
sudo systemctl status keepalived

# Monitor VRRP state on master
ip addr show | grep 192.168.100.10
journalctl -u keepalived -f

# Test failover by stopping master
sudo systemctl stop keepalived
# OR simulate network failure
sudo iptables -A OUTPUT -p vrrp -j DROP

# Verify backup takes over
ping -c 5 192.168.100.10
curl http://192.168.100.10/

# Monitor state transitions
tail -f /var/log/keepalived-state.log
grep -i "state" /var/log/syslog | grep keepalived

# Install HAProxy alongside keepalived
sudo apt install haproxy -y
sudo systemctl enable haproxy

# Backup original configuration
sudo cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.backup

cat << 'EOF' | sudo tee /etc/haproxy/haproxy.cfg
global
    log 127.0.0.1:514 local1 info
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

defaults
    mode http
    log global
    option httplog
    option dontlognull
    option redispatch
    option forwardfor
    retries 3
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms

listen stats
    bind *:8404
    stats enable
    stats uri /haproxy-stats
    stats refresh 30s
    stats admin if TRUE

frontend web_frontend
    bind 192.168.100.10:80
    bind 192.168.100.10:443 ssl crt /etc/ssl/certs/server.pem
    redirect scheme https if !{ ssl_fc }
    default_backend web_servers

backend web_servers
    balance roundrobin
    option httpchk GET /health
    http-check expect status 200
    server web1 192.168.100.20:80 check inter 2000ms fall 3 rise 2
    server web2 192.168.100.21:80 check inter 2000ms fall 3 rise 2
    server web3 192.168.100.22:80 check inter 2000ms fall 3 rise 2

backend db_servers
    mode tcp
    balance leastconn
    option tcp-check
    tcp-check connect
    server db1 192.168.100.30:3306 check inter 5000ms fall 2 rise 1
    server db2 192.168.100.31:3306 check inter 5000ms fall 2 rise 1 backup
EOF

cat << 'EOF' | sudo tee /etc/keepalived/scripts/check_haproxy.sh
#!/bin/bash

HAPROXY_PID_FILE="/var/run/haproxy.pid"
HAPROXY_CONFIG="/etc/haproxy/haproxy.cfg"
LOG_FILE="/var/log/keepalived-haproxy-check.log"

log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
}

# Check if HAProxy process is running
if [ -f "$HAPROXY_PID_FILE" ]; then
    HAPROXY_PID=$(cat "$HAPROXY_PID_FILE")
    if ! kill -0 "$HAPROXY_PID" 2>/dev/null; then
        log_message "HAProxy process not running (PID: $HAPROXY_PID)"
        # Attempt to start HAProxy
        systemctl start haproxy
        sleep 3
    fi
else
    log_message "HAProxy PID file not found"
    systemctl start haproxy
    sleep 3
fi

# Test HAProxy stats endpoint
if curl -f http://localhost:8404/haproxy-stats >/dev/null 2>&1; then
    # Check backend server health
    STATS_OUTPUT=$(echo "show stat" | socat stdio /run/haproxy/admin.sock)
    BACKEND_UP=$(echo "$STATS_OUTPUT" | grep -c ",UP,")
    
    if [ "$BACKEND_UP" -gt 0 ]; then
        log_message "HAProxy healthy - $BACKEND_UP backends UP"
        exit 0
    else
        log_message "No backend servers available"
        exit 1
    fi
else
    log_message "HAProxy stats endpoint unreachable"
    exit 1
fi
EOF

sudo chmod +x /etc/keepalived/scripts/check_haproxy.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/check_haproxy.sh

# Install Nginx with stream module
sudo apt install nginx-full -y

# Configure Nginx for load balancing
cat << 'EOF' | sudo tee /etc/nginx/sites-available/loadbalancer
upstream web_backend {
    least_conn;
    server 192.168.100.20:80 max_fails=3 fail_timeout=30s;
    server 192.168.100.21:80 max_fails=3 fail_timeout=30s;
    server 192.168.100.22:80 max_fails=3 fail_timeout=30s;
}

upstream api_backend {
    ip_hash;
    server 192.168.100.25:8080 max_fails=2 fail_timeout=15s;
    server 192.168.100.26:8080 max_fails=2 fail_timeout=15s;
}

server {
    listen 192.168.100.10:80;
    server_name example.com www.example.com;
    
    location /health {
        access_log off;
        return 200 "healthy\n";
        add_header Content-Type text/plain;
    }
    
    location /api/ {
        proxy_pass http://api_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_connect_timeout 5s;
        proxy_send_timeout 10s;
        proxy_read_timeout 10s;
    }
    
    location / {
        proxy_pass http://web_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_connect_timeout 5s;
        proxy_send_timeout 10s;
        proxy_read_timeout 10s;
    }
}
EOF

# Enable the configuration
sudo ln -sf /etc/nginx/sites-available/loadbalancer /etc/nginx/sites-enabled/
sudo nginx -t && sudo systemctl reload nginx

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id LB_PRIMARY
    script_user keepalived_script
    enable_script_security
    vrrp_strict
    vrrp_garp_interval 0
    vrrp_gna_interval 0
}

vrrp_script chk_nginx {
    script "/etc/keepalived/scripts/check_nginx_lb.sh"
    interval 2
    weight -3
    fall 2
    rise 2
    user keepalived_script
}

vrrp_script chk_backend_health {
    script "/etc/keepalived/scripts/check_backend_health.sh"
    interval 5
    weight -2
    fall 3
    rise 2
    user keepalived_script
}

vrrp_instance VI_LOADBALANCER {
    state MASTER
    interface ens192
    virtual_router_id 52
    priority 120
    advert_int 1
    preempt_delay 10
    authentication {
        auth_type AH
        auth_pass LBCluster2024
    }
    virtual_ipaddress {
        192.168.100.10/24 dev ens192 label ens192:vip
    }
    virtual_routes {
        192.168.100.0/24 dev ens192
    }
    track_script {
        chk_nginx
        chk_backend_health
    }
    notify_master "/etc/keepalived/scripts/lb_master.sh"
    notify_backup "/etc/keepalived/scripts/lb_backup.sh"
    notify_fault "/etc/keepalived/scripts/lb_fault.sh"
}
EOF

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id MULTI_INSTANCE_01
    script_user keepalived_script
    enable_script_security
}

# Web service VRRP instance
vrrp_instance VI_WEB {
    state MASTER
    interface ens192
    virtual_router_id 60
    priority 110
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass WebCluster2024
    }
    virtual_ipaddress {
        192.168.100.10/24 dev ens192
    }
    track_script {
        chk_web_service
    }
}

# Database service VRRP instance
vrrp_instance VI_DATABASE {
    state BACKUP
    interface ens192
    virtual_router_id 61
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass DbCluster2024
    }
    virtual_ipaddress {
        192.168.100.15/24 dev ens192
    }
    track_script {
        chk_db_service
    }
}

# API service VRRP instance
vrrp_instance VI_API {
    state MASTER
    interface ens192
    virtual_router_id 62
    priority 105
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ApiCluster2024
    }
    virtual_ipaddress {
        192.168.100.20/24 dev ens192
    }
    track_script {
        chk_api_service
    }
    notify_master "/etc/keepalived/scripts/api_master.sh"
    notify_backup "/etc/keepalived/scripts/api_backup.sh"
}
EOF

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id UNICAST_NODE_01
    vrrp_strict
    vrrp_skip_check_adv_addr
}

vrrp_instance VI_UNICAST {
    state MASTER
    interface ens192
    virtual_router_id 70
    priority 110
    advert_int 1
    
    # Unicast configuration
    unicast_src_ip 192.168.100.100
    unicast_peer {
        192.168.100.101
        192.168.100.102
        192.168.100.103
    }
    
    authentication {
        auth_type PASS
        auth_pass UnicastSecure2024
    }
    
    virtual_ipaddress {
        192.168.100.50/24
    }
    
    # Advanced timing options
    preempt_delay 30
    garp_master_delay 5
    garp_master_refresh 60
}
EOF

cat << 'EOF' | sudo tee -a /etc/keepalived/keepalived.conf
vrrp_sync_group VG_SERVICES {
    group {
        VI_WEB
        VI_DATABASE
        VI_API
    }
    notify_master "/etc/keepalived/scripts/sync_group_master.sh"
    notify_backup "/etc/keepalived/scripts/sync_group_backup.sh"
    notify_fault "/etc/keepalived/scripts/sync_group_fault.sh"
    smtp_alert
}

# Track interface for sync group
vrrp_instance VI_WEB {
    # ... existing configuration ...
    track_interface {
        ens192 weight -10
        ens193 weight -5
    }
}
EOF

# Generate strong authentication keys
VRRP_KEY=$(openssl rand -hex 20)
echo "Generated VRRP Key: $VRRP_KEY" | sudo tee /etc/keepalived/auth.key

cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id SECURE_NODE_01
    enable_script_security
    script_user keepalived_script
    vrrp_strict
    vrrp_iptables
}

vrrp_instance VI_SECURE {
    state MASTER
    interface ens192
    virtual_router_id 80
    priority 110
    advert_int 1
    
    # AH authentication (more secure than PASS)
    authentication {
        auth_type AH
        auth_pass SecureCluster2024AH
    }
    
    virtual_ipaddress {
        192.168.100.80/24
    }
    
    # Security enhancements
    dont_track_primary
    preempt
    preempt_delay 60
    
    # Custom timers for stability
    garp_master_delay 10
    garp_master_refresh 30
}
EOF

# Secure the configuration file
sudo chmod 600 /etc/keepalived/keepalived.conf
sudo chown root:root /etc/keepalived/keepalived.conf

cat << 'EOF' | sudo tee /etc/keepalived/scripts/advanced_health_check.sh
#!/bin/bash

LOG_FILE="/var/log/keepalived-advanced-health.log"
WEIGHT_ADJUSTMENT=0

log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE"
}

# Check service availability
check_service_health() {
    local service_name="$1"
    local endpoint="$2"
    local expected_response="$3"
    
    if systemctl is-active "$service_name" >/dev/null 2>&1; then
        if curl -f -s "$endpoint" | grep -q "$expected_response"; then
            log_message "$service_name: Healthy"
            return 0
        else
            log_message "$service_name: Service running but endpoint unhealthy"
            return 1
        fi
    else
        log_message "$service_name: Service not running"
        return 2
    fi
}

# Check load average
check_system_load() {
    local load_1min=$(uptime | awk '{print $10}' | sed 's/,//')
    local cpu_count=$(nproc)
    local load_threshold=$(echo "$cpu_count * 2" | bc)
    
    if (( $(echo "$load_1min > $load_threshold" | bc -l) )); then
        log_message "High system load: $load_1min (threshold: $load_threshold)"
        WEIGHT_ADJUSTMENT=$((WEIGHT_ADJUSTMENT - 20))
    fi
}

# Check memory usage
check_memory_usage() {
    local mem_usage=$(free | awk 'NR==2{printf "%.0f", $3*100/$2}')
    
    if [ "$mem_usage" -gt 90 ]; then
        log_message "High memory usage: $mem_usage%"
        WEIGHT_ADJUSTMENT=$((WEIGHT_ADJUSTMENT - 10))
    fi
}

# Perform health checks
check_service_health "nginx" "http://localhost:80/health" "healthy"
NGINX_STATUS=$?

check_service_health "haproxy" "http://localhost:8404/haproxy-stats" "Statistics Report"
HAPROXY_STATUS=$?

check_system_load
check_memory_usage

# Determine exit status based on checks
if [ $NGINX_STATUS -eq 0 ] && [ $HAPROXY_STATUS -eq 0 ]; then
    log_message "All services healthy (weight adjustment: $WEIGHT_ADJUSTMENT)"
    exit 0
elif [ $NGINX_STATUS -eq 1 ] || [ $HAPROXY_STATUS -eq 1 ]; then
    log_message "Some services degraded"
    exit 1
else
    log_message "Critical services failed"
    exit 2
fi
EOF

sudo chmod +x /etc/keepalived/scripts/advanced_health_check.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/advanced_health_check.sh

# Watch VRRP instance status
watch -n 1 'ip addr show | grep -E "(inet.*192\.168\.100|ens192)"'

# Monitor keepalived logs in real-time
sudo tail -f /var/log/syslog | grep keepalived
journalctl -u keepalived -f --since "1 hour ago"

# Check VRRP advertisement packets
sudo tcpdump -i ens192 -n proto 112

# Monitor process status
ps aux | grep -E "(keepalived|haproxy|nginx)" | grep -v grep
watch -n 2 'systemctl status keepalived --no-pager -l'

# Check VRRP instance statistics
sudo cat /proc/net/vrrp
cat /sys/class/net/ens192/operstate

# Monitor network interface statistics
ip -s link show ens192
ethtool -S ens192 | grep -E "(rx_|tx_|drop|error)"

# Check ARP table for virtual IP
ip neighbor show | grep 192.168.100.10
cat /proc/net/arp | grep 192.168.100.10

# Verify routing table
ip route show table main | grep 192.168.100
ip route show table local | grep 192.168.100

cat << 'EOF' | sudo tee /etc/keepalived/scripts/monitor_keepalived.sh
#!/bin/bash

MONITOR_LOG="/var/log/keepalived-monitor.log"
ALERT_EMAIL="admin@company.com"
HOSTNAME=$(hostname)
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

log_message() {
    echo "[$TIMESTAMP] $1" | tee -a "$MONITOR_LOG"
}

send_alert() {
    local subject="$1"
    local message="$2"
    echo "$message" | mail -s "$subject" "$ALERT_EMAIL"
    log_message "ALERT SENT: $subject"
}

# Check keepalived process
check_keepalived_process() {
    if ! pgrep -x keepalived >/dev/null; then
        send_alert "Keepalived Process Down on $HOSTNAME" \
                   "Keepalived process is not running on $HOSTNAME at $TIMESTAMP"
        return 1
    fi
    return 0
}

# Check VRRP instance state
check_vrrp_state() {
    local vip="$1"
    local expected_state="$2"
    
    if ip addr show | grep -q "$vip"; then
        if [ "$expected_state" = "BACKUP" ]; then
            send_alert "Unexpected MASTER State on $HOSTNAME" \
                       "$HOSTNAME has VIP $vip but should be in BACKUP state"
        fi
        log_message "VIP $vip is assigned (MASTER state)"
    else
        if [ "$expected_state" = "MASTER" ]; then
            send_alert "Expected MASTER State Lost on $HOSTNAME" \
                       "$HOSTNAME lost VIP $vip but should be MASTER"
        fi
        log_message "VIP $vip is not assigned (BACKUP/FAULT state)"
    fi
}

# Check VRRP advertisement reception
check_vrrp_advertisements() {
    local interface="$1"
    local timeout=10
    
    log_message "Checking VRRP advertisements on $interface..."
    
    if timeout "$timeout" tcpdump -i "$interface" -c 5 -n proto 112 >/dev/null 2>&1; then
        log_message "VRRP advertisements detected on $interface"
        return 0
    else
        send_alert "No VRRP Advertisements on $HOSTNAME" \
                   "No VRRP advertisements detected on $interface for $timeout seconds"
        return 1
    fi
}

# Check service health
check_service_health() {
    local service="$1"
    
    if ! systemctl is-active "$service" >/dev/null; then
        send_alert "Service $service Down on $HOSTNAME" \
                   "Service $service is not active on $HOSTNAME"
        return 1
    fi
    
    log_message "Service $service is active"
    return 0
}

# Main monitoring checks
log_message "Starting keepalived monitoring on $HOSTNAME"

check_keepalived_process
check_vrrp_state "192.168.100.10" "MASTER"  # Adjust expected state as needed
check_vrrp_advertisements "ens192"
check_service_health "nginx"
check_service_health "haproxy"

# Check configuration file syntax
if ! keepalived -t -f /etc/keepalived/keepalived.conf >/dev/null 2>&1; then
    send_alert "Keepalived Configuration Error on $HOSTNAME" \
               "Configuration file syntax error detected on $HOSTNAME"
fi

log_message "Monitoring check completed"
EOF

sudo chmod +x /etc/keepalived/scripts/monitor_keepalived.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/monitor_keepalived.sh

# Schedule monitoring every 5 minutes
echo "*/5 * * * * /etc/keepalived/scripts/monitor_keepalived.sh" | sudo crontab -u root -

# Check for configuration syntax errors
sudo keepalived -t -f /etc/keepalived/keepalived.conf

# Debug VRRP communication issues
sudo tcpdump -i ens192 -v proto 112 | head -20
netstat -s | grep -i vrrp

# Verify firewall rules for VRRP
sudo iptables -L -n | grep -E "(112|vrrp)"
sudo firewall-cmd --list-all | grep -i vrrp

# Check interface configuration
ip addr show ens192 | grep -E "(inet|state)"
ethtool ens192 | grep -E "(Link detected|Speed|Duplex)"

# Examine system logs for errors
journalctl -u keepalived --since "24 hours ago" | grep -i error
dmesg | grep -E "(keepalived|vrrp|network)" | tail -10

# Test virtual IP connectivity
ping -c 3 -W 2 192.168.100.10
arping -c 3 -I ens192 192.168.100.10

# Check for split-brain scenarios
for host in 192.168.100.100 192.168.100.101; do
    echo "Checking $host:"
    ssh "$host" "ip addr show | grep 192.168.100.10"
done

# Monitor VRRP performance metrics
cat << 'EOF' | sudo tee /etc/keepalived/scripts/performance_monitor.sh
#!/bin/bash

LOG_FILE="/var/log/keepalived-performance.log"

# Log VRRP statistics
echo "$(date): VRRP Performance Metrics" >> "$LOG_FILE"
cat /proc/net/vrrp >> "$LOG_FILE"

# Monitor network interface performance
echo "Network Interface Statistics:" >> "$LOG_FILE"
cat /proc/net/dev | grep ens192 >> "$LOG_FILE"

# Check keepalived memory usage
KEEPALIVED_PID=$(pgrep keepalived)
if [ -n "$KEEPALIVED_PID" ]; then
    ps -o pid,ppid,cmd,%mem,%cpu -p "$KEEPALIVED_PID" >> "$LOG_FILE"
fi

# Monitor system resources
echo "System Load: $(uptime | awk '{print $10}' | sed 's/,//')" >> "$LOG_FILE"
echo "Memory Usage: $(free | awk 'NR==2{printf "%.1f%%", $3*100/$2}')" >> "$LOG_FILE"
echo "---" >> "$LOG_FILE"
EOF

sudo chmod +x /etc/keepalived/scripts/performance_monitor.sh
sudo chown keepalived_script:keepalived_script /etc/keepalived/scripts/performance_monitor.sh

# Schedule performance monitoring every 10 minutes
echo "*/10 * * * * /etc/keepalived/scripts/performance_monitor.sh" | sudo crontab -u keepalived_script -

# Configure iptables for VRRP security
sudo iptables -A INPUT -p vrrp -s 192.168.100.0/24 -j ACCEPT
sudo iptables -A INPUT -p vrrp -j DROP

# Allow keepalived management traffic
sudo iptables -A INPUT -p tcp --dport 22 -s 192.168.100.0/24 -j ACCEPT
sudo iptables -A INPUT -p udp --dport 161 -s 192.168.100.0/24 -j ACCEPT

# Log dropped VRRP packets for monitoring
sudo iptables -A INPUT -p vrrp -j LOG --log-prefix "VRRP-DROP: "

# Save iptables rules
sudo iptables-save > /etc/iptables/rules.v4

# Create custom VRRP service
sudo firewall-cmd --permanent --new-service=vrrp
sudo firewall-cmd --permanent --service=vrrp --set-description="VRRP protocol for keepalived"
sudo firewall-cmd --permanent --service=vrrp --add-protocol=vrrp

# Configure rich rules for VRRP security
sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="192.168.100.0/24" service name="vrrp" accept'
sudo firewall-cmd --permanent --add-rich-rule='rule family="ipv4" service name="vrrp" drop'

# Enable and reload firewall
sudo firewall-cmd --reload
sudo firewall-cmd --list-all

# Generate secure VRRP passwords
VRRP_PASS=$(openssl rand -base64 32 | cut -c1-20)
echo "Generated secure password: $VRRP_PASS" | sudo tee /etc/keepalived/vrrp_auth.key

# Create dedicated keepalived user with minimal privileges
sudo useradd -r -s /bin/false -d /var/lib/keepalived keepalived_script
sudo mkdir -p /var/lib/keepalived /var/run/keepalived
sudo chown keepalived_script:keepalived_script /var/lib/keepalived /var/run/keepalived

# Configure sudo access for scripts
cat << 'EOF' | sudo tee /etc/sudoers.d/keepalived
# Allow keepalived_script user to manage specific services
keepalived_script ALL=(ALL) NOPASSWD: /bin/systemctl start nginx
keepalived_script ALL=(ALL) NOPASSWD: /bin/systemctl stop nginx
keepalived_script ALL=(ALL) NOPASSWD: /bin/systemctl start haproxy
keepalived_script ALL=(ALL) NOPASSWD: /bin/systemctl stop haproxy
keepalived_script ALL=(ALL) NOPASSWD: /sbin/ip addr *
keepalived_script ALL=(ALL) NOPASSWD: /sbin/ip route *
EOF

sudo visudo -c -f /etc/sudoers.d/keepalived

# Set restrictive permissions on configuration files
sudo chmod 600 /etc/keepalived/keepalived.conf
sudo chown root:root /etc/keepalived/keepalived.conf

# Secure script directory
sudo chmod 750 /etc/keepalived/scripts
sudo chown root:keepalived_script /etc/keepalived/scripts

# Set secure permissions for individual scripts
sudo find /etc/keepalived/scripts -name "*.sh" -exec chmod 750 {} \;
sudo find /etc/keepalived/scripts -name "*.sh" -exec chown root:keepalived_script {} \;

# Create and secure log directory
sudo mkdir -p /var/log/keepalived
sudo chmod 755 /var/log/keepalived
sudo chown keepalived_script:keepalived_script /var/log/keepalived

# Set up log rotation
cat << 'EOF' | sudo tee /etc/logrotate.d/keepalived
/var/log/keepalived/*.log {
    daily
    missingok
    rotate 14
    compress
    delaycompress
    notifempty
    create 644 keepalived_script keepalived_script
    postrotate
        /bin/systemctl reload keepalived > /dev/null 2>&1 || true
    endscript
}
EOF

# Configure VLAN for keepalived management
sudo ip link add link ens192 name ens192.100 type vlan id 100
sudo ip addr add 10.100.1.10/24 dev ens192.100
sudo ip link set ens192.100 up

# Update keepalived configuration for VLAN
cat << 'EOF' | sudo tee /etc/keepalived/keepalived.conf
global_defs {
    router_id SECURE_VLAN_NODE
    script_user keepalived_script
    enable_script_security
    vrrp_strict
    vrrp_iptables
}

vrrp_instance VI_SECURE {
    state MASTER
    interface ens192.100  # Use VLAN interface
    virtual_router_id 90
    priority 110
    advert_int 1
    
    # Unicast for added security
    unicast_src_ip 10.100.1.10
    unicast_peer {
        10.100.1.11
        10.100.1.12
    }
    
    authentication {
        auth_type AH
        auth_pass SecureVLAN2024
    }
    
    virtual_ipaddress {
        10.100.1.100/24 dev ens192.100
    }
}
EOF

cat << 'EOF' | sudo tee /etc/keepalived/scripts/security_monitor.sh
#!/bin/bash

SECURITY_LOG="/var/log/keepalived-security.log"
ALERT_EMAIL="security@company.com"

log_security_event() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [SECURITY] $1" | tee -a "$SECURITY_LOG"
}

send_security_alert() {
    local subject="$1"
    local message="$2"
    echo "$message" | mail -s "$subject" "$ALERT_EMAIL"
    log_security_event "SECURITY ALERT: $subject"
}

# Check for unauthorized VRRP traffic
check_vrrp_sources() {
    local authorized_sources="192.168.100.100 192.168.100.101 192.168.100.102"
    local unauthorized_found=0
    
    # Capture VRRP packets for analysis
    timeout 10 tcpdump -i ens192 -n -q proto 112 2>/dev/null | while read line; do
        source_ip=$(echo "$line" | awk '{print $3}' | cut -d: -f1)
        if [[ ! " $authorized_sources " =~ " $source_ip " ]]; then
            send_security_alert "Unauthorized VRRP Source Detected" \
                                "Unauthorized VRRP traffic from $source_ip detected on $(hostname)"
            unauthorized_found=1
        fi
    done
}

# Monitor configuration file changes
check_config_integrity() {
    local config_file="/etc/keepalived/keepalived.conf"
    local checksum_file="/var/lib/keepalived/config.sha256"
    
    if [ -f "$checksum_file" ]; then
        if ! sha256sum -c "$checksum_file" >/dev/null 2>&1; then
            send_security_alert "Keepalived Configuration Modified" \
                                "Unauthorized modification of $config_file detected on $(hostname)"
        fi
    else
        sha256sum "$config_file" > "$checksum_file"
    fi
}

# Check for suspicious process activity
check_process_integrity() {
    local keepalived_pids=$(pgrep keepalived)
    
    for pid in $keepalived_pids; do
        local cmd=$(ps -p "$pid" -o cmd=)
        if [[ ! "$cmd" =~ ^/usr/sbin/keepalived ]]; then
            send_security_alert "Suspicious Keepalived Process" \
                                "Unexpected keepalived process detected: $cmd"
        fi
    done
}

# Main security monitoring
log_security_event "Starting security monitoring"
check_vrrp_sources
check_config_integrity
check_process_integrity
log_security_event "Security monitoring completed"
EOF

sudo chmod +x /etc/keepalived/scripts/security_monitor.sh
sudo chown root:keepalived_script /etc/keepalived/scripts/security_monitor.sh

# Schedule security monitoring
echo "*/15 * * * * /etc/keepalived/scripts/security_monitor.sh" | sudo crontab -u root -

# Generate SSL certificates for keepalived communication
sudo mkdir -p /etc/keepalived/ssl
cd /etc/keepalived/ssl

# Create CA certificate
sudo openssl genrsa -out ca.key 4096
sudo openssl req -new -x509 -days 3650 -key ca.key -out ca.crt \
    -subj "/C=US/ST=State/L=City/O=Organization/OU=IT/CN=KeepalivedCA"

# Generate node certificates
for node in master backup1 backup2; do
    sudo openssl genrsa -out ${node}.key 2048
    sudo openssl req -new -key ${node}.key -out ${node}.csr \
        -subj "/C=US/ST=State/L=City/O=Organization/OU=IT/CN=${node}"
    sudo openssl x509 -req -days 365 -in ${node}.csr -CA ca.crt -CAkey ca.key \
        -CAcreateserial -out ${node}.crt
done

# Set secure permissions
sudo chmod 600 /etc/keepalived/ssl/*.key
sudo chmod 644 /etc/keepalived/ssl/*.crt
sudo chown -R root:keepalived_script /etc/keepalived/ssl

# Check priority configuration on all nodes
grep -r "priority" /etc/keepalived/keepalived.conf

# Verify preemption settings
grep -r "nopreempt\|preempt" /etc/keepalived/keepalived.conf

# Monitor election process
journalctl -u keepalived -f | grep -i "election\|priority\|master"

# Force master election by restarting keepalived
sudo systemctl restart keepalived

# Check interface configuration
ip addr show | grep -A5 -B5 ens192

# Verify VRRP instance state
journalctl -u keepalived --since "10 minutes ago" | grep -i "state\|vip"

# Test manual IP assignment
sudo ip addr add 192.168.100.10/24 dev ens192 label ens192:test
sudo ip addr del 192.168.100.10/24 dev ens192

# Check for IP conflicts
arping -c 3 -I ens192 192.168.100.10

# Verify identical passwords across nodes
for node in master backup1 backup2; do
    ssh "$node" "grep auth_pass /etc/keepalived/keepalived.conf"
done

# Check authentication type consistency
grep -A3 -B1 "authentication" /etc/keepalived/keepalived.conf

# Monitor authentication errors
journalctl -u keepalived | grep -i "auth\|password"

# Restart with new authentication
sudo systemctl restart keepalived

# Check VIP assignment across cluster
for node in 192.168.100.100 192.168.100.101 192.168.100.102; do
    echo "Node $node:"
    ssh "$node" "ip addr show | grep 192.168.100.10"
done

# Monitor VRRP advertisements from all nodes
sudo tcpdump -i ens192 -n proto 112 | grep -E "192\.168\.100\.(100|101|102)"

# Implement unicast to prevent multicast issues
# Update configuration with unicast_src_ip and unicast_peer

# Add network monitoring scripts
/etc/keepalived/scripts/check_split_brain.sh

# Test health check scripts manually
sudo -u keepalived_script /etc/keepalived/scripts/check_nginx.sh
echo "Exit code: $?"

# Adjust health check sensitivity
# Increase fall count and interval in vrrp_script

# Debug health check logging
tail -f /var/log/keepalived-health-check.log

# Verify service dependencies
systemctl list-dependencies nginx.service
systemctl status nginx.service

# Check current firewall rules
sudo iptables -L -n | grep -E "(112|vrrp)"
sudo firewall-cmd --list-all | grep vrrp

# Temporarily disable firewall for testing
sudo systemctl stop firewalld
sudo systemctl start keepalived

# Add proper VRRP rules
sudo firewall-cmd --permanent --add-protocol=vrrp
sudo firewall-cmd --reload

# Monitor dropped packets
sudo iptables -A INPUT -p vrrp -j LOG --log-prefix "VRRP: "
sudo tail -f /var/log/messages | grep "VRRP:"