I/O Redirection and Pipes

# View file descriptors for a process
ls -la /proc/$$/fd/
# 0 -> /dev/pts/0
# 1 -> /dev/pts/0
# 2 -> /dev/pts/0
# 255 -> /dev/pts/0 (script file)

# View limits
cat /proc/$$/limits | grep "open files"

# Truncate and write stdout
command > file.txt
command 1> file.txt

# Append stdout
command >> file.txt
command 1>> file.txt

# Redirect stderr
command 2> error.log

# Append stderr
command 2>> error.log

# Redirect both stdout and stderr (POSIX)
command > output.log 2>&1

# Redirect both (bash shorthand)
command &> output.log
command &>> output.log

# Discard output
command > /dev/null
command 2> /dev/null
command &> /dev/null

# Read stdin from file
command < input.txt
command 0< input.txt

# Read from here-document
command << EOF
line 1
line 2
$VARIABLE expanded here
EOF

# Read from here-document with tab stripping
command <<- EOF
    tab-indented content
    (tabs stripped, spaces preserved)
EOF

# Read from here-string
command <<< "single line of input"
grep "error" <<< "this line has an error"

# Open file for reading and writing (<>)
exec 3<> file.txt
read -r line <&3
echo "new line" >&3
exec 3<&-

# WRONG: stderr redirect sees the original stdout (terminal)
command 2>&1 > file.txt
# stderr goes to terminal, stdout goes to file.txt

# CORRECT: stdout redirect happens first, then stderr copies it
command > file.txt 2>&1
# Both stdout and stderr go to file.txt

# Redirect specific command in a pipeline
ls -la /nonexistent 2>&1 | grep -i "no such"

# Redirect a block of code
{
    echo "start"
    ls -la /nonexistent
    echo "end"
} > output.log 2>&1

# Redirect a function
myfunc() {
    echo "stdout from function"
    echo "stderr from function" >&2
}
myfunc > func_out.log 2> func_err.log

# Redirect a loop
for file in *.log; do
    wc -l "$file"
done > line_counts.txt

# Basic pipe — stdout of left goes to stdin of right
command1 | command2

# Pipeline return code is the exit status of the last command
false | true
echo $?    # 0 (true"s exit code)

# With pipefail, pipeline fails if ANY command fails
set -o pipefail
false | true
echo $?    # 1 (false's exit code)

# View pipe buffer size
cat /proc/sys/fs/pipe-max-size
# 1048576 (1 MiB on modern Linux)

# The default buffer is 64 KiB (65536 bytes) since kernel 2.6.11
# When the buffer is full, the writer blocks

# Increase pipe buffer size (requires CAP_SYS_RESOURCE)
# This can improve throughput in high-volume pipelines
dd if=/dev/urandom bs=1M count=100 | md5sum

# Demonstrate SIGPIPE
yes | head -n 5
# "yes" writes infinitely; "head" reads 5 lines and closes stdin
# "yes" receives SIGPIPE and terminates

# Ignore SIGPIPE (useful in network programming)
trap '' PIPE
# Now write to a closed pipe returns EPIPE instead of killing the process

# Check for broken pipe in scripts
yes | head -n 5; echo "exit: $?"

# Create a named pipe
mkfifo /tmp/my_pipe

# In terminal 1: write to the pipe (blocks until reader connects)
echo "hello from writer" > /tmp/my_pipe

# In terminal 2: read from the pipe
cat < /tmp/my_pipe

# The pipe is unidirectional by default
# For bidirectional communication, use two pipes
mkfifo /tmp/pipe_in /tmp/pipe_out

# 1. Simple IPC between processes
mkfifo /tmp/cmd_pipe
# Writer:
while true; do
    read -r cmd
    echo "Processing: $cmd"
done < /tmp/cmd_pipe
# Reader (in another terminal):
echo "status" > /tmp/cmd_pipe
echo "restart" > /tmp/cmd_pipe

# 2. Sequential processing
mkfifo /tmp/buffer
sort data.txt > /tmp/buffer &
uniq < /tmp/buffer > /tmp/buffer2 &
awk '{print $2}' < /tmp/buffer2

# 3. Log multiplexing
mkfifo /tmp/log_pipe
tail -f /tmp/log_pipe | while read -r line; do
    echo "$(date) $line" >> /var/log/app.log
    echo "$line" | grep -i "error" >> /var/log/errors.log
done &

# Send logs to the pipe
app1 --log /tmp/log_pipe &
app2 --log /tmp/log_pipe &

# 4. Progress monitoring
mkfifo /tmp/progress
# Long-running process writes progress
for i in $(seq 1 100); do
    echo "$i"
    sleep 0.1
done > /tmp/progress
# Monitor reads progress
tail -f /tmp/progress

# Check pipe status
ls -la /tmp/my_pipe
# prw-r--r-- 1 user user 0 ...

# Named pipes have zero size
du /tmp/my_pipe    # 0

# Named pipes persist until deleted
rm /tmp/my_pipe

# Multiple readers: only one gets each message
# Multiple writers: messages may interleave

# Compare output of two commands
diff <(sort file1.txt) <(sort file2.txt)

# Pass command output to a command expecting a file argument
wc -l <(find /etc -name "*.conf")
md5sum <(tar cf - /home/user/docs)

# Multiple inputs
paste <(cut -f1 data.txt) <(cut -f2 data.txt)

# Feed grep patterns from command output
grep -f <(echo -e "error\nwarning\ncritical") /var/log/syslog

# Split stdout and stderr to different processes
command > >(grep "INFO" >> info.log) 2> >(grep "ERROR" >> error.log)

# Parallel processing
cat largefile.txt | tee >(process1 &>/dev/null) | process2

# Background logging
my_long_command > >(while read -r line; do
    echo "$(date) $line"
done >> /var/log/mycommand.log) 2>&1

# Write to stdout and a file
command | tee output.log

# Append mode
command | tee -a output.log

# Write to multiple files
command | tee file1.log file2.log file3.log

# Discard stdout, write only to files
command | tee output.log > /dev/null

# With sudo (write to root-owned files)
command | sudo tee /etc/config.conf

# tee with pipes
command | tee /dev/tty | grep error

# Interactive monitoring
dmesg | tee /tmp/dmesg.log

# 1. Log and display simultaneously
./deploy.sh 2>&1 | tee /var/log/deploy-$(date +%Y%m%d-%H%M%S).log

# 2. Audit trail
sudo iptables-save | tee /etc/iptables/rules.v4 | iptables-restore

# 3. Multi-stream logging
tail -f /var/log/syslog | tee >(grep error >> errors.log) >(grep warn >> warnings.log) > /dev/null

# 4. Checkpoint processing
cat large_input | tee /tmp/checkpoint | process_data > output

# Build arguments from stdin
echo "file1.txt file2.txt file3.txt" | xargs rm
find /tmp -name "*.tmp" -print0 | xargs -0 rm

# Limit arguments per invocation
echo {1..100} | tr ' ' '\n' | xargs -n 5 echo

# Interactive mode (confirm each operation)
find . -name "*.log" | xargs -p rm

# Run 4 processes in parallel
find . -name "*.jpg" | xargs -P 4 -I {} convert {} -resize 50% {}_small.jpg

# Parallel downloads
cat urls.txt | xargs -P 8 -I {} wget -q {}

# Parallel compression
find . -name "*.log" -print0 | xargs -0 -P 4 -I {} gzip {}

# Run with progress
cat files.txt | xargs -P 4 -I {} sh -c 'echo "Processing {}"; process "{}"'

# -I {} — replace string
echo "a b c" | xargs -I {} echo "item: {}"
# item: a b c

# -n N — max arguments per command
seq 1 10 | xargs -n 3 echo
# 1 2 3
# 4 5 6
# 7 8 9
# 10

# -0 — null-delimited input (safe for filenames with spaces/newlines)
find . -print0 | xargs -0 -n 1 echo

# -d DELIM — custom delimiter
echo "a:b:c" | xargs -d ': " -I {} echo "item: {}"

# --max-procs or -P — parallel execution
seq 1 20 | xargs -P 4 -I {} sleep 1 && echo {}

# -L N — max lines per command
cat addresses.txt | xargs -L 1 curl -s -o /dev/null -w "%{http_code}\n"

# xargs — faster, handles argument limits
find . -name "*.log" | xargs gzip

# for loop — safer, easier to add logic
find . -name "*.log" -print0 | while IFS= read -r -d ''" file; do
    echo "Compressing $file"
    gzip "$file"
done