Collecting Incident Artifacts

#!/usr/bin/env bash
# collect-incident-artifacts.sh
# Usage: ./collect-incident-artifacts.sh [output_dir]
set -euo pipefail

OUT="${1:-/tmp/cassandra-incident}"
NODE=$(hostname -s)
TS=$(date +%Y%m%dT%H%M%S)
DIR="${OUT}/${NODE}-${TS}"
mkdir -p "$DIR"

CASS_LOG_DIR="${CASSANDRA_LOG_DIR:-/var/log/cassandra}"
CASS_CONF_DIR="${CASSANDRA_CONF:-/etc/cassandra}"

echo "[collect] Writing to $DIR"

# --- Cluster state ---
echo "[collect] Cluster state"
nodetool status        > "$DIR/nodetool-status.txt"       2>&1 || true
nodetool info          > "$DIR/nodetool-info.txt"          2>&1 || true
nodetool describecluster > "$DIR/nodetool-describecluster.txt" 2>&1 || true
nodetool ring          > "$DIR/nodetool-ring.txt"          2>&1 || true
nodetool gossipinfo    > "$DIR/nodetool-gossipinfo.txt"    2>&1 || true

# --- Performance ---
echo "[collect] Performance metrics"
nodetool tpstats       > "$DIR/nodetool-tpstats.txt"       2>&1 || true
nodetool proxyhistograms > "$DIR/nodetool-proxyhistograms.txt" 2>&1 || true
nodetool compactionstats -H > "$DIR/nodetool-compactionstats.txt" 2>&1 || true

# --- JVM state ---
echo "[collect] JVM state"
CASS_PID=$(pgrep -f CassandraDaemon 2>/dev/null || echo "")
if [ -n "$CASS_PID" ]; then
    jcmd "$CASS_PID" Thread.print      > "$DIR/thread-dump-1.txt" 2>&1 || true
    sleep 10
    jcmd "$CASS_PID" Thread.print      > "$DIR/thread-dump-2.txt" 2>&1 || true
    sleep 10
    jcmd "$CASS_PID" Thread.print      > "$DIR/thread-dump-3.txt" 2>&1 || true
    jcmd "$CASS_PID" GC.heap_info      > "$DIR/heap-info.txt"     2>&1 || true
    jcmd "$CASS_PID" VM.flags          > "$DIR/jvm-flags.txt"     2>&1 || true
    jcmd "$CASS_PID" VM.system_properties > "$DIR/jvm-system-props.txt" 2>&1 || true
else
    echo "WARNING: CassandraDaemon process not found" > "$DIR/jvm-state-warning.txt"
fi

# --- Logs (last 24 hours) ---
echo "[collect] Logs"
LOG_CUTOFF=$(date -d '24 hours ago' '+%Y-%m-%d %H:%M:%S' 2>/dev/null \
    || date -v-24H '+%Y-%m-%d %H:%M:%S')  # macOS fallback
if [ -f "$CASS_LOG_DIR/system.log" ]; then
    cp "$CASS_LOG_DIR/system.log" "$DIR/system.log"
fi
if ls "$CASS_LOG_DIR"/gc.log* 2>/dev/null | head -1 | grep -q .; then
    cp "$CASS_LOG_DIR"/gc.log* "$DIR/"
fi

# --- Configuration (raw, redact before sharing) ---
echo "[collect] Configuration (NOT YET REDACTED)"
for f in cassandra.yaml jvm-server.options jvm11-server.options \
          jvm8-server.options cassandra-env.sh logback.xml; do
    [ -f "$CASS_CONF_DIR/$f" ] && cp "$CASS_CONF_DIR/$f" "$DIR/$f-RAW" || true
done

# --- OS context ---
echo "[collect] OS context"
uname -a               > "$DIR/os-uname.txt"    2>&1 || true
free -h                > "$DIR/os-memory.txt"   2>&1 || true
df -h                  > "$DIR/os-disk.txt"     2>&1 || true
iostat -xz 1 5         > "$DIR/os-iostat.txt"   2>&1 || true
top -bn1               > "$DIR/os-top.txt"      2>&1 || true

echo "[collect] Done. Output: $DIR"
echo "[collect] IMPORTANT: Redact $DIR/*.yaml-RAW and $DIR/*.options-RAW before sharing."

DIR=/tmp/cassandra-incident/node1-20260331T143000

# Write a summary
cat > "$DIR/incident-summary.txt" <<'EOF'
Cluster: production-us-east
Nodes affected: node1, node2
Incident window: 2026-03-31 14:00--14:45 UTC
Symptom: p99 read latency > 30 s; ReadTimeout on keyspace orders
Cassandra version: 6.0.0  JDK: OpenJDK 17.0.10
EOF

tar -czf cassandra-incident-node1-20260331T143000.tar.gz -C /tmp/cassandra-incident node1-20260331T143000/

CREATE ROLE incident_collector WITH PASSWORD = 'change-me' AND LOGIN = true;
GRANT SELECT ON ALL TABLES IN KEYSPACE system_views TO incident_collector;
GRANT SELECT ON ALL TABLES IN KEYSPACE system_metrics TO incident_collector;