{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":466,"y":32,"width":199.64236000000002,"height":223.20000000000002,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":486,"y":50,"width":159.64236000000002,"height":187.20000000000002,"fontSize":18,"fontFamily":1,"text":"Ingestion Worker\n\n• Parse tags\n• Resolve\nseries_id\n• Write to TSDB\n• Update tag\nindex","originalText":"Ingestion Worker\n\n• Parse tags\n• Resolve\nseries_id\n• Write to TSDB\n• Update tag\nindex","textAlign":"center","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":36,"y":32,"width":390,"height":316.8,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":56,"y":50,"width":350,"height":280.8,"fontSize":18,"fontFamily":1,"text":"COLLECTION LAYER\n\n[Host Agents] --→ [Ingestion\nGateway (50 nodes)] --→ [Kafka]\n[k8s Exporters]-→ • Auth + rate\nlimit\n[Custom SDKs] --→ • 10M points/sec\ncapacity\n[Prometheus ]-→ • Push + pull\nsupport [20 brokers,\n[ remote-write] • 202 Accepted\n(async) 3× replication]","originalText":"COLLECTION LAYER\n\n[Host Agents] --→ [Ingestion\nGateway (50 nodes)] --→ [Kafka]\n[k8s Exporters]-→ • Auth + rate\nlimit\n[Custom SDKs] --→ • 10M points/sec\ncapacity\n[Prometheus ]-→ • Push + pull\nsupport [20 brokers,\n[ remote-write] • 202 Accepted\n(async) 3× replication]","textAlign":"center","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":36,"y":388.79999999999995,"width":192,"height":176.4,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":56,"y":406.79999999999995,"width":152,"height":140.4,"fontSize":18,"fontFamily":1,"text":"Alert Router\n\n• Match point\nto rules\n• Fan-out to\nevaluators","originalText":"Alert Router\n\n• Match point\nto rules\n• Fan-out to\nevaluators","textAlign":"center","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":268,"y":388.79999999999995,"width":192,"height":199.8,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":288,"y":406.79999999999995,"width":152,"height":163.8,"fontSize":18,"fontFamily":1,"text":"Rollup Worker\n\n• Read raw\nblocks past\nretention\n• Compute agg\n• Write rollup","originalText":"Rollup Worker\n\n• Read raw\nblocks past\nretention\n• Compute agg\n• Write rollup","textAlign":"center","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":36,"y":628.5999999999999,"width":390,"height":550.8000000000001,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":56,"y":646.5999999999999,"width":350,"height":514.8000000000001,"fontSize":18,"fontFamily":1,"text":"STORAGE LAYER\n\n• Time-Series DB (TSDB) (100\nstorage nodes)\n• Sharded by series_id\n• WAL + in-memory active\n• 2-hour immutable blocks\n• Gorilla compression\n• Raw: 15d, 1m: 3mo,\n1h: 2yr, 1d: forever\n• 2 replicas per shard\n\n• Tag Inverted Index (20 nodes)\n• Roaring Bitmaps\n• tag=value → series_ids\n• Bitmap AND/OR for queries\n• ~200 GB total\n\n• Alert State (Redis Cluster)\n• Sliding windows per rule\n• Alert status per rule\n• Notification dedup","originalText":"STORAGE LAYER\n\n• Time-Series DB (TSDB) (100\nstorage nodes)\n• Sharded by series_id\n• WAL + in-memory active\n• 2-hour immutable blocks\n• Gorilla compression\n• Raw: 15d, 1m: 3mo,\n1h: 2yr, 1d: forever\n• 2 replicas per shard\n\n• Tag Inverted Index (20 nodes)\n• Roaring Bitmaps\n• tag=value → series_ids\n• Bitmap AND/OR for queries\n• ~200 GB total\n\n• Alert State (Redis Cluster)\n• Sliding windows per rule\n• Alert status per rule\n• Notification dedup","textAlign":"center","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":466,"y":628.5999999999999,"width":325.74208000000004,"height":597.6,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":486,"y":646.5999999999999,"width":285.74208000000004,"height":561.6,"fontSize":18,"fontFamily":1,"text":"QUERY & ALERT LAYER\n\n• Query Engine (50 nodes)\n• Parse query language\n• Resolve tags → series\n• Auto-select rollup tier\n• Scatter to TSDB nodes\n• Gather + aggregate\n• < 500 ms response\n\n• Alert Evaluators (50 nodes)\n• 5M rules, ~83K evals/sec\n• Threshold, anomaly, SLO\n• State machine per rule\nOK → WARN → ALERT → RECOV\nNotification Router\n• PagerDuty, Slack, email\n• Dedup + rate limit\n• Escalation policies\n\n• Notification Router\n• PagerDuty, Slack, email\n• Dedup + rate limit\n• Escalation policies","originalText":"QUERY & ALERT LAYER\n\n• Query Engine (50 nodes)\n• Parse query language\n• Resolve tags → series\n• Auto-select rollup tier\n• Scatter to TSDB nodes\n• Gather + aggregate\n• < 500 ms response\n\n• Alert Evaluators (50 nodes)\n• 5M rules, ~83K evals/sec\n• Threshold, anomaly, SLO\n• State machine per rule\nOK → WARN → ALERT → RECOV\nNotification Router\n• PagerDuty, Slack, email\n• Dedup + rate limit\n• Escalation policies\n\n• Notification Router\n• PagerDuty, Slack, email\n• Dedup + rate limit\n• Escalation policies","textAlign":"center","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":36,"y":1266.1999999999998,"width":309.92476000000005,"height":340.2,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":56,"y":1284.1999999999998,"width":269.92476000000005,"height":304.2,"fontSize":18,"fontFamily":1,"text":"PRESENTATION LAYER\n\n• Dashboard Service\n• Widget rendering\n• Auto-refresh (10-60s)\n• Template variables\n• Sharing + permissions\n\n• API (Public)\n• Submit metrics\n• Query metrics\n• CRUD monitors/dashboards\n• SLO management","originalText":"PRESENTATION LAYER\n\n• Dashboard Service\n• Widget rendering\n• Auto-refresh (10-60s)\n• Template variables\n• Sharing + permissions\n\n• API (Public)\n• Submit metrics\n• Query metrics\n• CRUD monitors/dashboards\n• SLO management","textAlign":"center","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":385.9247600000001,"y":1266.1999999999998,"width":390,"height":784.8000000000001,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":405.9247600000001,"y":1284.1999999999998,"width":350,"height":748.8000000000001,"fontSize":18,"fontFamily":1,"text":"Key Self-Metrics:\n• Ingestion rate (points/sec, by\ntenant)\n• Ingestion lag (Kafka consumer\noffset lag)\n• TSDB write latency (p50, p99)\n• Query latency (p50, p99) — target\n< 500ms\n• Active series count (total, by\ntenant)\n• Alert evaluation lag (time from\ndata → evaluation)\n• Alert notification delivery latency\n• Storage utilization per TSDB node\n• Compression ratio (actual vs\ntheoretical)\n• Rollup pipeline lag\n\nAlerts (on the monitoring system\nitself):\nRED — Ingestion lag > 5 min (data\nis stale)\nRED — Alert evaluation lag > 2 min\n(alerts are delayed)\nRED — TSDB node unreachable\nRED — Dead-man's switch not\nreceived (meta-monitoring failure)\nWARN — Query p99 > 2 seconds\nWARN — Disk utilization > 80% on\nany TSDB node\nWARN — Per-tenant series count\napproaching quota","originalText":"Key Self-Metrics:\n• Ingestion rate (points/sec, by\ntenant)\n• Ingestion lag (Kafka consumer\noffset lag)\n• TSDB write latency (p50, p99)\n• Query latency (p50, p99) — target\n< 500ms\n• Active series count (total, by\ntenant)\n• Alert evaluation lag (time from\ndata → evaluation)\n• Alert notification delivery latency\n• Storage utilization per TSDB node\n• Compression ratio (actual vs\ntheoretical)\n• Rollup pipeline lag\n\nAlerts (on the monitoring system\nitself):\nRED — Ingestion lag > 5 min (data\nis stale)\nRED — Alert evaluation lag > 2 min\n(alerts are delayed)\nRED — TSDB node unreachable\nRED — Dead-man's switch not\nreceived (meta-monitoring failure)\nWARN — Query p99 > 2 seconds\nWARN — Disk utilization > 80% on\nany TSDB node\nWARN — Per-tenant series count\napproaching quota","textAlign":"center","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":36,"y":2091,"width":420,"height":68,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":56,"y":2113.95,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"[Kafka: metrics-raw topic]","originalText":"[Kafka: metrics-raw topic]","textAlign":"left","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":36,"y":2183,"width":493.16696,"height":68,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":56,"y":2205.95,"width":453.16696,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Monitoring the Monitoring System (Meta-Monitoring):","originalText":"Monitoring the Monitoring System (Meta-Monitoring):","textAlign":"left","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}