Golden Signals and Alerting

High coordinator read latency
├── Are pending compaction tasks > 50?
│   └── YES → Compaction backlog: throttle background work, add I/O capacity
│               See: operate/compaction-overview.adoc
├── Is heap used > 80%?
│   └── YES → GC pressure: tune -Xmx, audit tombstone reads, check row cache
├── Are read timeouts increasing?
│   ├── YES + unavailables > 0 → Node(s) unreachable: check nodetool status, gossip
│   └── YES + unavailables = 0 → Slow replica: check per-node latency, hot partition
├── Is read throughput flat or increasing?
│   └── YES + latency rising → Working-set growth: partition size, secondary indexes
└── Is read throughput dropping?
    └── YES → Client back-off or connection issues: check driver metrics

Dropped mutations > 0
├── Is MutationStage.PendingTasks > 1000?
│   └── YES → Write thread pool saturated
│       ├── Is disk I/O at capacity? → Reduce flush frequency or add IOPS
│       └── Is GC pausing > 200ms? → Tune heap, switch to G1 or ZGC
├── Is commit log segment count rising?
│   └── YES → Commit log stalling: check commit log directory disk latency
├── Are batch mutations in use?
│   └── YES → Oversized batches: reduce batch size, enable batch_size_warn_threshold
└── Is the node under heavy compaction?
    └── YES → Compaction stealing I/O: adjust compaction throughput

Unavailables > 0 (any count)
├── Run: nodetool status
│   ├── Any node DN (Down/Normal)? → Node failure: investigate system.log, restart if safe
│   └── Any node DN (Down/Leaving)? → Decommission stalled: check decommission status
├── Are multiple DCs in use?
│   └── YES + LOCAL_QUORUM used → Check inter-DC reachability, network partition
├── Is this a CAS (LWT) operation?
│   └── YES → Paxos unavailables: check all replicas for the token range are UP
└── Check gossip: nodetool gossipinfo | grep STATUS
    └── Any BOOT or LEAVING? → Ring change in progress, wait or intervene

Heap used > 80%
├── Is row cache enabled?
│   └── YES → Row cache over-consuming: reduce row_cache_size_in_mb
├── Are tombstone warnings in system.log?
│   └── YES → Tombstone reads materializing large result sets: fix data model or TTL
├── Is off-heap overhead high?
│   └── Check: bloom filters, compression metadata, key cache sizes
└── Is GC pausing > 500ms repeatedly?
    └── YES → Switch to G1GC or ZGC, increase -Xmx if heap budget allows

-javaagent:/opt/cassandra/agents/jmx_prometheus_javaagent.jar=7070:/etc/cassandra/jmx_exporter.yaml

startDelaySeconds: 0
ssl: false
rules:
  # Coordinator request latency and rates (latency + throughput)
  - pattern: 'org.apache.cassandra.metrics<type=ClientRequest, scope=(\w+), name=(Latency|Timeouts|Unavailables)><>(Count|OneMinuteRate|99thPercentile)'
    name: cassandra_client_request_$2_$3
    labels:
      request_type: "$1"
    attrNameSnakeCase: true

  # Dropped messages (errors)
  - pattern: 'org.apache.cassandra.metrics<type=DroppedMessage, scope=(\w+), name=Dropped><>(Count|OneMinuteRate)'
    name: cassandra_dropped_message_$2
    labels:
      message_type: "$1"

  # Thread pool pending and blocked (saturation)
  - pattern: 'org.apache.cassandra.metrics<type=ThreadPools, path=(\w+), scope=(\w+), name=(PendingTasks|BlockedTasks)><>Value'
    name: cassandra_thread_pool_$3
    labels:
      pool_type: "$1"
      pool_name: "$2"
    attrNameSnakeCase: true

  # Compaction pending (saturation)
  - pattern: 'org.apache.cassandra.metrics<type=Compaction, name=PendingTasks><>Value'
    name: cassandra_compaction_pending_tasks

  # JVM heap (saturation)
  - pattern: 'java.lang<type=Memory><HeapMemoryUsage>(\w+)'
    name: jvm_memory_heap_$1

scrape_configs:
  - job_name: cassandra
    static_configs:
      - targets:
          - cassandra-node-1:7070
          - cassandra-node-2:7070
          - cassandra-node-3:7070
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        regex: '([^:]+):\d+'
        replacement: '$1'
    scrape_interval: 15s
    scrape_timeout: 10s

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cassandra-golden-signals
  namespace: monitoring
  labels:
    prometheus: kube-prometheus
    role: alert-rules
spec:
  groups:
    - name: cassandra.latency
      interval: 30s
      rules:
        - alert: CassandraHighReadLatencyWarning
          expr: |
            cassandra_client_request_latency_99thpercentile{request_type="Read"} > 10000
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Cassandra coordinator read p99 latency above 10ms"
            description: "Node {{ $labels.instance }} read p99 is {{ $value | printf \"%.0f\" }} µs."

        - alert: CassandraHighReadLatencyCritical
          expr: |
            cassandra_client_request_latency_99thpercentile{request_type="Read"} > 50000
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Cassandra coordinator read p99 latency above 50ms"
            description: "Node {{ $labels.instance }} read p99 is {{ $value | printf \"%.0f\" }} µs."

        - alert: CassandraHighWriteLatencyCritical
          expr: |
            cassandra_client_request_latency_99thpercentile{request_type="Write"} > 25000
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "Cassandra coordinator write p99 latency above 25ms"
            description: "Node {{ $labels.instance }} write p99 is {{ $value | printf \"%.0f\" }} µs."

    - name: cassandra.errors
      interval: 30s
      rules:
        - alert: CassandraDroppedMutations
          expr: |
            rate(cassandra_dropped_message_count{message_type="MUTATION"}[5m]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Cassandra is dropping mutation messages"
            description: "Node {{ $labels.instance }} dropped mutations at {{ $value | humanize }}/s."

        - alert: CassandraUnavailableWrites
          expr: |
            increase(cassandra_client_request_unavailables_count{request_type="Write"}[5m]) > 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "Cassandra write unavailables detected"
            description: "Node {{ $labels.instance }} had {{ $value }} unavailable write exceptions."

    - name: cassandra.saturation
      interval: 30s
      rules:
        - alert: CassandraCompactionBacklogWarning
          expr: |
            cassandra_compaction_pending_tasks > 20
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Cassandra compaction backlog building"
            description: "Node {{ $labels.instance }} has {{ $value }} pending compaction tasks."

        - alert: CassandraCompactionBacklogCritical
          expr: |
            cassandra_compaction_pending_tasks > 100
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "Cassandra compaction backlog critical"
            description: "Node {{ $labels.instance }} has {{ $value }} pending compaction tasks."

        - alert: CassandraHeapHighWarning
          expr: |
            jvm_memory_heap_used / jvm_memory_heap_max > 0.75
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "Cassandra JVM heap usage above 75%"
            description: "Node {{ $labels.instance }} heap is at {{ $value | humanizePercentage }}."