Linux Log Analysis: Guide for System Troubleshooting

# View recent system logs in real-time
journalctl -f

# Search for specific errors in all system logs
journalctl --priority=err --since "1 hour ago"

# Analyze authentication failures quickly
grep "Failed" /var/log/auth.log | tail -20

# Check which logging system is active
systemctl status systemd-journald
systemctl status rsyslog

# View journal storage information
journalctl --disk-usage
journalctl --verify

# Check journal configuration
cat /etc/systemd/journald.conf

# Examine kernel messages with priority filtering
dmesg --level=err,warn --human --decode

# View systemd service hierarchy
systemctl list-units --type=service --state=failed

# Check boot sequence timing
systemd-analyze blame
systemd-analyze critical-chain

# Standard log analysis workflow
# 1. Identify time range of interest
journalctl --since "2025-12-16 10:00" --until "2025-12-16 11:00"

# 2. Filter by priority and service
journalctl -p err -u nginx.service --since "1 hour ago"

# 3. Examine specific log patterns
journalctl | grep -i "failed\|error\|critical" | tail -50

# Monitor critical system logs simultaneously
tail -f /var/log/syslog /var/log/kern.log /var/log/auth.log

# Use journalctl for modern systems
journalctl -f --priority=warning

# Check multiple services together
journalctl -f -u ssh.service -u nginx.service -u mysql.service

# Comprehensive security log analysis
# Check recent authentication failures
grep "Failed password" /var/log/auth.log | tail -20

# Analyze successful logins
last -n 20

# Examine sudo usage
grep "sudo:" /var/log/auth.log | tail -10

# Check for unusual user activity
who -a

# Web server log analysis
# Apache access patterns
tail -f /var/log/apache2/access.log | awk '{print $1, $7, $9}' | sort | uniq -c

# Nginx error investigation
grep -E "(error|warn)" /var/log/nginx/error.log | tail -20

# Database log examination
journalctl -u mysql.service --since "today" --priority=err

# Mail server log analysis
grep "postfix" /var/log/mail.log | tail -30

# Advanced journalctl filtering techniques
# Time-based analysis with specific formats
journalctl --since "2025-12-16 09:00:00" --until "2025-12-16 17:00:00"

# Service-specific detailed analysis
journalctl -u systemd-resolved.service --output=json-pretty

# Boot-specific log examination
journalctl -b -1 --priority=err

# Follow logs with context
journalctl -f --lines=50 --output=short-iso

# Sophisticated grep log analysis patterns
# Multi-pattern log searching
grep -E "(failed|error|critical|timeout)" /var/log/syslog | grep -v "systemd"

# Context-aware error analysis
grep -A 3 -B 3 "ERROR" /var/log/application.log

# Time-stamped pattern matching
grep "$(date '+%Y-%m-%d')" /var/log/syslog | grep -i error

# Case-insensitive comprehensive search
grep -ri "out of memory" /var/log/

# Powerful awk log analysis scripts
# Apache log analysis with statistics
awk '{print $1}' /var/log/apache2/access.log | sort | uniq -c | sort -nr | head -20

# System load analysis from logs
awk '/load average/ {print $NF}' /var/log/syslog | sort -n

# Memory usage tracking from logs
awk '/MemAvailable/ {print $2, $3}' /proc/meminfo

# Custom timestamp analysis
awk '{print $1, $2, $3}' /var/log/syslog | sort | uniq -c | tail -20

# Effective sed log analysis operations
# Remove timestamps for pattern analysis
sed 's/^[A-Za-z]* [0-9]* [0-9:]*//g' /var/log/syslog | sort | uniq -c

# Extract specific log sections
sed -n '/ERROR/,/INFO/p' /var/log/application.log

# Clean log format for analysis
sed 's/\[.*\]//' /var/log/nginx/error.log | grep -v "^$"

# Convert log timestamps
sed 's/\([0-9]\{4\}\)-\([0-9]\{2\}\)-\([0-9]\{2\}\)/\3-\2-\1/g' logfile.log

# Systematic initial log assessment
# 1. Establish timeline boundaries
echo "Current system time: $(date)"
echo "Last boot time: $(uptime -s)"

# 2. Check system resource status
df -h | grep -E "(9[0-9]%|100%)"
free -h
cat /proc/loadavg

# 3. Identify active services with issues
systemctl --failed
systemctl list-units --state=failed

# Pattern recognition analysis workflow
# 1. Identify frequent error patterns
grep -E "(error|failed|critical)" /var/log/syslog | awk '{print $5}' | sort | uniq -c | sort -nr

# 2. Analyze timing patterns
journalctl --since "today" | awk '{print $1, $2, $3}' | uniq -c | sort -nr

# 3. Service-specific pattern analysis
journalctl -u nginx.service | grep -E "(40[0-9]|50[0-9])" | awk '{print $1, $2, $3}' | uniq -c

# Advanced log correlation analysis
# 1. Multi-source timestamp correlation
# Create unified timeline from multiple sources
(
  journalctl -u nginx.service --since "1 hour ago" --output=short-iso | sed 's/^/nginx: /'
  journalctl -u mysql.service --since "1 hour ago" --output=short-iso | sed 's/^/mysql: /'
  grep "$(date '+%Y-%m-%d %H:')" /var/log/apache2/error.log | sed 's/^/apache: /'
) | sort

# 2. Event sequence analysis
journalctl --since "1 hour ago" | grep -E "(started|stopped|failed)" | sort

# Installing essential log analysis tools
# 1. Install multitail for multiple file monitoring
sudo apt-get install multitail

# 2. Install lnav for enhanced log navigation
sudo apt-get install lnav

# 3. Install goaccess for web log analysis
sudo apt-get install goaccess

# 4. Install jq for JSON log processing
sudo apt-get install jq

# Using specialized log analysis tools
# 1. Web server log analysis with goaccess
goaccess /var/log/nginx/access.log --log-format=COMBINED -o report.html

# 2. Enhanced log viewing with lnav
lnav /var/log/syslog /var/log/auth.log

# 3. Multiple file monitoring with multitail
multitail /var/log/syslog /var/log/auth.log /var/log/nginx/error.log

# 4. JSON log analysis with jq
journalctl -o json | jq '.MESSAGE' | head -10

# Performance-oriented log analysis
# 1. System performance from logs
dmesg | grep -E "(OOM|memory|CPU)"

# 2. I/O performance analysis from logs
journalctl | grep -E "(I/O error|timeout|slow)"

# 3. Network performance log analysis
grep -E "(timeout|connection.*failed)" /var/log/syslog

# 4. Application performance metrics
journalctl -u application.service | grep -E "(slow|timeout|performance)"

#!/bin/bash
# Comprehensive log monitoring automation script

# Function: Critical error detection
check_critical_errors() {
    local error_count=$(journalctl --since "5 minutes ago" --priority=err | wc -l)
    if [ "$error_count" -gt 5 ]; then
        echo "ALERT: $error_count critical errors detected in last 5 minutes"
        journalctl --since "5 minutes ago" --priority=err
    fi
}

# Function: Authentication failure monitoring
check_auth_failures() {
    local failed_logins=$(grep "Failed password" /var/log/auth.log | grep "$(date '+%b %d %H:%M')" | wc -l)
    if [ "$failed_logins" -gt 10 ]; then
        echo "SECURITY ALERT: $failed_logins failed login attempts detected"
        grep "Failed password" /var/log/auth.log | tail -10
    fi
}

# Function: Disk space monitoring from logs
check_disk_warnings() {
    local disk_warnings=$(journalctl --since "10 minutes ago" | grep -i "no space\|disk full" | wc -l)
    if [ "$disk_warnings" -gt 0 ]; then
        echo "STORAGE ALERT: Disk space warnings detected"
        df -h | grep -E "(9[0-9]%|100%)"
    fi
}

# Execute monitoring functions
check_critical_errors
check_auth_failures
check_disk_warnings

# Real-time log monitoring implementation
# 1. Continuous error monitoring
journalctl -f --priority=err | while read line; do
    echo "$(date): ERROR DETECTED - $line"
    # Add alerting logic here
done &

# 2. Authentication monitoring daemon
tail -f /var/log/auth.log | while read line; do
    if echo "$line" | grep -q "Failed password"; then
        echo "$(date): Authentication failure detected"
        # Add security response logic here
    fi
done &

# 3. Service failure monitoring
systemctl --failed --no-legend | while read line; do
    echo "$(date): Service failure detected - $line"
    # Add service recovery logic here
done

# Scheduled log analysis cron jobs
# Add to crontab: crontab -e

# Daily comprehensive log summary (runs at 6 AM)
# 0 6 * * * /home/admin/scripts/daily_log_analysis.sh

#!/bin/bash
# daily_log_analysis.sh - Comprehensive daily log analysis

LOG_DATE=$(date '+%Y-%m-%d')
REPORT_FILE="/var/reports/daily_log_analysis_$LOG_DATE.txt"

{
    echo "Daily Log Analysis Report - $LOG_DATE"
    echo "======================================="
    
    echo "System Errors (Last 24 Hours):"
    journalctl --since "24 hours ago" --priority=err | wc -l
    
    echo "Authentication Summary:"
    grep "$(date '+%b %d')" /var/log/auth.log | grep "session opened" | wc -l
    
    echo "Service Status:"
    systemctl --failed --no-legend | wc -l
    
    echo "Disk Usage Warnings:"
    journalctl --since "24 hours ago" | grep -i "disk\|space" | wc -l
    
} > "$REPORT_FILE"

# Email report to administrators
mail -s "Daily Log Analysis - $LOG_DATE" admin@company.com < "$REPORT_FILE"

# Advanced multi-source log correlation
# 1. Time-synchronized correlation across services
{
    echo "=== Web Server Events ==="
    journalctl -u nginx.service --since "1 hour ago" --output=short-iso
    echo "=== Database Events ==="
    journalctl -u postgresql.service --since "1 hour ago" --output=short-iso
    echo "=== System Events ==="
    journalctl --since "1 hour ago" --priority=warning --output=short-iso
} | sort | head -50

# 2. Error correlation analysis
grep -h "$(date '+%Y-%m-%d %H:')" /var/log/nginx/error.log /var/log/postgresql/postgresql-*.log | sort

# Event sequence correlation methodology
# 1. Create unified timeline with source identification
create_unified_timeline() {
    local time_range="$1"
    
    # Combine multiple log sources with source tags
    (
        journalctl -u nginx.service --since "$time_range" --output=short-iso | sed 's/^/WEB: /'
        journalctl -u mysql.service --since "$time_range" --output=short-iso | sed 's/^/DB: /'
        journalctl --priority=err --since "$time_range" --output=short-iso | sed 's/^/SYS: /'
    ) | sort -k2,3
}

# Usage example
create_unified_timeline "2 hours ago" | head -30

# Cross-system correlation framework
# 1. Remote log collection for correlation
collect_remote_logs() {
    local remote_host="$1"
    local time_range="$2"
    
    ssh "$remote_host" "journalctl --since '$time_range' --output=short-iso" | \
    sed "s/^/[$remote_host] /"
}

# 2. Multi-host correlation analysis
analyze_distributed_logs() {
    local time_range="1 hour ago"
    
    {
        echo "Local system logs:"
        journalctl --since "$time_range" --output=short-iso | sed 's/^/[LOCAL] /'
        
        echo "Remote system logs:"
        collect_remote_logs "server1.example.com" "$time_range"
        collect_remote_logs "server2.example.com" "$time_range"
    } | sort -k2,3 | head -50
}

# Advanced statistical log analysis
# 1. Error rate trend analysis
analyze_error_trends() {
    local days="$1"
    
    for i in $(seq 0 "$days"); do
        local date=$(date -d "$i days ago" '+%Y-%m-%d')
        local errors=$(journalctl --since "$date 00:00:00" --until "$date 23:59:59" --priority=err | wc -l)
        echo "$date: $errors errors"
    done
}

# 2. Service availability analysis
analyze_service_availability() {
    local service="$1"
    local hours="$2"
    
    echo "Service: $service - Last $hours hours"
    for i in $(seq 0 "$hours"); do
        local hour=$(date -d "$i hours ago" '+%Y-%m-%d %H:00:00')
        local next_hour=$(date -d "$((i-1)) hours ago" '+%Y-%m-%d %H:00:00')
        local downtime=$(journalctl -u "$service" --since "$hour" --until "$next_hour" | grep -c "stopped\|failed")
        echo "Hour $hour: $downtime incidents"
    done
}

# Usage examples
analyze_error_trends 7
analyze_service_availability "nginx.service" 24

# Advanced pattern mining techniques
# 1. Frequent pattern identification
identify_frequent_patterns() {
    local logfile="$1"
    local min_frequency="$2"
    
    # Extract and count common patterns
    awk '{for(i=1;i<=NF;i++) print $i}' "$logfile" | \
    sort | uniq -c | sort -nr | \
    awk -v freq="$min_frequency" '$1 >= freq {print}'
}

# 2. Error pattern classification
classify_error_patterns() {
    journalctl --priority=err --since "24 hours ago" | \
    grep -oE '[A-Z][a-z]+ [a-z]+|[Ee]rror [0-9]+|[Ff]ailed.*$' | \
    sort | uniq -c | sort -nr | head -20
}

# 3. User activity pattern analysis
analyze_user_patterns() {
    last | awk '{print $1, $4, $5}' | grep -v "reboot\|wtmp" | \
    sort | uniq -c | sort -nr | head -15
}

# Execute pattern analysis
classify_error_patterns
analyze_user_patterns

# Predictive log analysis implementation
# 1. Disk space growth prediction
predict_disk_usage() {
    local partition="$1"
    local days="$2"
    
    echo "Disk usage trend for $partition over last $days days:"
    for i in $(seq "$days" -1 0); do
        local date=$(date -d "$i days ago" '+%Y-%m-%d')
        local usage=$(grep "$date.*$partition" /var/log/syslog | grep -o '[0-9]\+%' | tail -1)
        echo "$date: ${usage:-N/A}"
    done
}

# 2. Service failure prediction
predict_service_failures() {
    local service="$1"
    local threshold="$2"
    
    local recent_failures=$(journalctl -u "$service" --since "7 days ago" | grep -c "failed\|error")
    local failure_rate=$((recent_failures / 7))
    
    if [ "$failure_rate" -gt "$threshold" ]; then
        echo "WARNING: $service showing elevated failure rate ($failure_rate/day)"
        echo "Recent failures:"
        journalctl -u "$service" --since "7 days ago" | grep "failed\|error" | tail -5
    fi
}

# 3. Resource exhaustion prediction
predict_resource_exhaustion() {
    echo "Resource utilization trends:"
    
    # Memory trend
    local mem_usage=$(grep "$(date '+%b %d')" /var/log/syslog | grep -i "memory" | wc -l)
    echo "Memory events today: $mem_usage"
    
    # CPU trend
    local cpu_usage=$(grep "$(date '+%b %d')" /var/log/syslog | grep -i "cpu\|load" | wc -l)
    echo "CPU events today: $cpu_usage"
}

# Execute predictive analysis
predict_service_failures "nginx.service" 2
predict_resource_exhaustion

# Troubleshooting log access issues
# 1. Check file permissions and ownership
ls -la /var/log/ | head -20

# 2. Verify user group membership
groups $(whoami)
id $(whoami)

# 3. Check SELinux context if applicable
ls -Z /var/log/secure 2>/dev/null || echo "SELinux not active"

# 4. Test log accessibility
if [ -r /var/log/syslog ]; then
    echo "syslog readable"
else
    echo "syslog access denied - check permissions"
fi

# Troubleshooting log rotation issues
# 1. Check logrotate configuration
cat /etc/logrotate.conf | grep -A 5 -B 5 "weekly\|daily\|monthly"

# 2. Verify logrotate service status
systemctl status logrotate.timer
journalctl -u logrotate.service

# 3. Find rotated/archived logs
find /var/log -name "*.gz" -mtime -7 | head -10
find /var/log -name "*.[0-9]" -mtime -7 | head -10

# 4. Check available log history
ls -la /var/log/syslog* | head -5

# Optimizing large log analysis performance
# 1. Check log file sizes
du -sh /var/log/* | sort -hr | head -10

# 2. Use efficient commands for large files
# Instead of: cat large.log | grep pattern
# Use: grep pattern large.log

# 3. Implement streaming analysis for real-time logs
tail -f /var/log/syslog | grep "pattern" | head -100

# 4. Use compression for archived analysis
zcat /var/log/syslog.*.gz | grep "pattern" | head -50