Log Files and Log Rotation

# Watch a log file in real time as new lines are appended
tail -f /var/log/syslog
tail -f /var/log/nginx/access.log

# Show the last 100 lines of a log
tail -n 100 /var/log/auth.log

# Follow multiple log files simultaneously
tail -f /var/log/syslog /var/log/auth.log

# Search for a pattern in a log file (case-insensitive)
grep -i "error" /var/log/syslog
grep -i "failed" /var/log/auth.log

# Search with context — show 3 lines before and after each match
grep -B 3 -A 3 "segfault" /var/log/kern.log

# Filter logs for a specific time window
grep "Mar 12 14:" /var/log/syslog        # all events at 14:xx on Mar 12
grep "Mar 12 1[4-6]:" /var/log/syslog    # 14:00 to 16:59

# Count occurrences of a pattern
grep -c "Failed password" /var/log/auth.log

# Find which IPs are generating the most failed SSH logins
grep "Failed password" /var/log/auth.log | \
  grep -oP '(\d{1,3}\.){3}\d{1,3}' | \
  sort | uniq -c | sort -rn | head -10

# View rsyslog routing rules
cat /etc/rsyslog.conf
ls /etc/rsyslog.d/

# Send a test message to syslog manually
logger "Test message from admin - $(date)"
logger -p auth.warning "Simulated auth warning"

# Verify the test message arrived
tail -5 /var/log/syslog

# Check rsyslog service status
sudo systemctl status rsyslog

# Reload rsyslog after config changes
sudo systemctl reload rsyslog

# Complete logrotate config for a custom application
sudo tee /etc/logrotate.d/myapp <<'EOF'
/var/log/myapp/*.log {
    daily
    rotate 14
    size 100M
    missingok
    notifempty
    compress
    delaycompress
    sharedscripts
    postrotate
        systemctl reload myapp 2>/dev/null || true
    endscript
}
EOF

# Test the config — debug mode shows what would happen without making changes
sudo logrotate -d /etc/logrotate.d/myapp

# Force an immediate rotation to test the config live
sudo logrotate -f /etc/logrotate.d/myapp

# Check the logrotate state file — records when each log was last rotated
cat /var/lib/logrotate/status | grep myapp

# Show all errors from the current boot
journalctl -p err -b 0

# Show kernel messages from the current boot (hardware/driver issues)
journalctl -k -b 0

# Show all logs for a specific user by UID
journalctl _UID=$(id -u alice)

# Show logs between two specific timestamps
journalctl --since "2025-03-12 14:00" --until "2025-03-12 15:00"

# Show logs for multiple units at once
journalctl -u nginx -u php-fpm --since "30 minutes ago"

# Extract structured fields with JSON output and jq
journalctl -u sshd -o json | jq -r 'select(.PRIORITY == "6") | .MESSAGE' | head -10

# List all unique systemd units that have journal entries
journalctl -F _SYSTEMD_UNIT | sort -u

# Show logs that match a specific executable
journalctl /usr/sbin/sshd

# Forward all logs to a remote syslog server via rsyslog
# Add to /etc/rsyslog.d/50-remote.conf

# Forward using UDP (faster, no delivery guarantee)
sudo tee /etc/rsyslog.d/50-remote.conf <<'EOF'
*.* @logs.example.com:514        # UDP
*.* @@logs.example.com:514       # TCP (reliable, use this for production)
EOF

sudo systemctl restart rsyslog

# Forward the systemd journal to a remote syslog server
# (requires rsyslog-journal module)
sudo tee -a /etc/rsyslog.conf <<'EOF'
module(load="imjournal")
*.* @@logs.example.com:514
EOF

# Test forwarding with logger
logger "Centralised log forwarding test"

# Common centralised logging destinations:
# - Elasticsearch + Kibana (ELK stack) — self-hosted, powerful
# - Grafana Loki — lightweight, integrates with Prometheus
# - Datadog / Splunk / Papertrail — managed SaaS solutions
# - CloudWatch Logs (AWS) / Cloud Logging (GCP) — cloud-native