Journal and Logging

# Restart rsyslog
systemctl restart rsyslog

# Check syntax
rsyslogd -N1    # config check mode

# Verify it is running
systemctl status rsyslog
ss -ulnp | grep 514

# Applications can log structured data via the journal
# Using sd_journal_send() in C:
# sd_journal_send("MESSAGE=Service started",
#                 "SERVICE_NAME=%s", "myapp",
#                 "VERSION=%s", "2.0.0",
#                 "PRIORITY=%i", LOG_INFO,
#                 NULL);

# Using systemd-cat with fields
systemd-cat -t myapp --identifier=myapp -p info echo "Service started"

# Using python-systemd
# from systemd import journal
# journal.send('MESSAGE=Hello', 'PRIORITY=6', 'MY_FIELD=my_value')

# Show specific fields
journalctl -u nginx -o json-pretty | \
    jq '{time: .__REALTIME_TIMESTAMP, message: .MESSAGE, priority: .PRIORITY}'

# Count errors by service
journalctl -p err --since yesterday -o json | \
    jq -r '._SYSTEMD_UNIT' | sort | uniq -c | sort -rn

# Find all messages with a specific field
journalctl _COMM=sshd -o verbose | grep "OBJECT_SYSTEMD_UNIT"

# Export specific fields as CSV
journalctl -u nginx --since today -o json | \
    jq -r '[.__REALTIME_TIMESTAMP, ._PID, .MESSAGE] | @csv'

# Check journal disk usage
journalctl --disk-usage

# Vacuum by size
journalctl --vacuum-size=500M

# Vacuum by time
journalctl --vacuum-time=30d

# Vacuum by number of files
journalctl --vacuum-files=20

# Show journal file sizes
ls -lhS /var/log/journal/*/system.journal

# Monitor disk usage
watch -n 60 'journalctl --disk-usage'

# Show the current boot's logs
journalctl -b

# Show boot timing (systemd-analyze)
systemd-analyze
# Startup finished in 2.341s (kernel) + 4.123s (userspace) = 6.464s

# Show the slowest services
systemd-analyze blame | head -20

# Critical chain (dependency chain that took longest)
systemd-analyze critical-chain

# Boot timeline
systemd-analyze plot > boot-timeline.svg

# Verify boot messages
journalctl -b -p err
journalctl -b --grep="error|failed|fatal"

# If /var/log/journal/ does not exist, logs are stored in RAM only
ls -la /var/log/journal/    # if this directory is missing, storage is volatile

# Fix: create the directory and restart journald
sudo mkdir -p /var/log/journal
sudo systemd-tmpfiles --create --prefix /var/log/journal
sudo systemctl restart systemd-journald

# Verify
journalctl --header | grep Storage
# Storage: persistent

# If a service produces too many messages, journald rate-limits it
journalctl -u noisy-service --since "1 hour ago" | wc -l
# Might show fewer messages than expected

# Check for rate-limiting messages
journalctl -u systemd-journald --grep="rate-limiting"

# Increase rate limits in journald.conf
# RateLimitIntervalSec=30s
# RateLimitBurst=100000

# By default, ForwardToSyslog=yes in journald.conf
# This means logs are stored in both the journal AND rsyslog files
# Double the disk usage

# Fix: disable forwarding if you only use the journal
# /etc/systemd/journald.conf
ForwardToSyslog=no

# Or: disable journal persistence if you only use rsyslog files
# /etc/systemd/journald.conf
Storage=volatile

# If a service runs in a container or is not managed by systemd,
# its logs may not appear in the journal

# Check if the service is systemd-managed
systemctl status myapp

# For non-systemd processes, logs go to:
# - Their own log files
# - /var/log/syslog (if rsyslog captures syslog() calls)
# - stdout/stderr is NOT captured if not started by systemd

# For scripts run by cron:
journalctl -u cron    # cron's own logs
# But the script's output is NOT in the journal unless it uses syslog()
# Fix: redirect cron job output
0 * * * * /usr/local/bin/myscript.sh 2>&1 | systemd-cat -t myscript

# Identify the largest journal files
journalctl --disk-usage
ls -lhS /var/log/journal/

# Find which units generate the most log data
journalctl --since "7 days ago" -o json | \
    jq -r '._SYSTEMD_UNIT // "kernel"' | \
    sort | uniq -c | sort -rn | head -20

# Fix: adjust the service's log level
# For systemd services, add:
# [Service]
# LogRateLimitIntervalSec=30s
# LogRateLimitBurst=10000

# For journald rate limiting, see journald.conf above

# journalctl uses the system timezone by default
timedatectl

# Show timestamps in UTC
journalctl --since today -o short-precise

# Show timestamps in a specific timezone
TZ=UTC journalctl --since today

# The journal stores timestamps in UTC internally
# Display format does not affect storage

# All error-level messages in the last 24 hours
journalctl --since yesterday -p err

# Errors grouped by service
journalctl --since yesterday -p err -o json | \
    jq -r '._SYSTEMD_UNIT // "kernel"' | \
    sort | uniq -c | sort -rn

# Errors with context (5 lines before and after)
journalctl --since yesterday -p err -B 5 -A 5

# Find recurring error patterns
journalctl --since "7 days ago" -p err -o cat | \
    sort | uniq -c | sort -rn | head -20

# Nginx: find all 5xx responses (if logging to journal)
journalctl -u nginx --since today -o cat | \
    awk '{print $NF}' | grep '^5' | wc -l

# SSH: find failed login attempts
journalctl -u sshd --since today -o cat | grep "Failed password"

# System: OOM killer events
journalctl -k --grep="Out of memory" --since "7 days ago"

# System: hardware errors
journalctl -k -p err --since "7 days ago"

# Create a timeline of significant events
journalctl --since "2026-04-06 08:00" --until "2026-04-06 10:00" \
    -p warning -o short-precise | \
    awk '{print $1, $2, $3}' | uniq -c

# Find what happened around a specific time
journalctl --since "2026-04-06 09:14:00" --until "2026-04-06 09:16:00" \
    -o verbose

# Systemd boot analysis
systemd-analyze
systemd-analyze blame | head -20
systemd-analyze critical-chain
systemd-analyze critical-chain --blur 0.1s

# Compare boot times between boots
systemd-analyze dump | grep 'FinishTimestamp'

# Identify services that slow down boot
systemd-analyze blame | awk '$1 > 1000 {print}'

# Generate boot chart (requires pybootchartgui or systemd-bootchart)
systemd-analyze plot > /tmp/boot-analysis.svg

# Size-based retention
journalctl --vacuum-size=1G

# Time-based retention
journalctl --vacuum-time=30d

# Combined: keep 30 days or 1G, whichever is smaller
journalctl --vacuum-time=30d --vacuum-size=1G

# Configure in journald.conf
# SystemMaxUse=1G
# MaxFileSec=1week

# journalctl can output JSON for parsing by exporters
# Common pattern: use promtail (Loki) or journald-exporter

# promtail config snippet
# /etc/promtail/config.yml
# scrape_configs:
#   - job_name: journal
#     journal:
#       max_age: 12h
#       labels:
#         job: systemd-journal
#     relabel_configs:
#       - source_labels: ['__journal__systemd_unit']
#         target_label: "unit''

# Good log messages include:
# - Timestamp
# - Log level
# - Service/component name
# - Correlation ID (for distributed tracing)
# - Structured key-value pairs

# Example using systemd-cat with structured fields
systemd-cat -t myapp -p info << "EOF'
message=Request completed
request_id=abc-123
duration_ms=42
status=200
EOF

# Python example (using systemd.journal)
# journal.send(
#     MESSAGE='Request completed',
#     SYSLOG_IDENTIFIER='myapp',
#     PRIORITY=6,  # info
#     REQUEST_ID='abc-123',
#     DURATION_MS='42',
#     STATUS='200'
# )

# Generate a correlation ID and propagate it through the log pipeline
CORRELATION_ID=$(uuidgen)

journalctl -u myapp -o json | \
    jq --arg cid "$CORRELATION_ID" 'select(.CORRELATION_ID == $cid)'