{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":264.8,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":34,"width":248,"height":228.8,"fontSize":22,"fontFamily":1,"text":"Ingestion Worker\n\n• Parse tags\n• Resolve\nseries_id\n• Write to TSDB\n• Update tag\nindex","originalText":"Ingestion Worker\n\n• Parse tags\n• Resolve\nseries_id\n• Write to TSDB\n• Update tag\nindex","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":324.8,"width":288,"height":550.8000000000001,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":342.8,"width":248,"height":514.8000000000001,"fontSize":22,"fontFamily":1,"text":"COLLECTION LAYER\n\n[Host Agents] --→\n[Ingestion Gateway\n(50 nodes)] --→\n[Kafka]\n[k8s Exporters]-→ •\nAuth + rate limit\n[Custom SDKs] --→ •\n10M points/sec\ncapacity\n[Prometheus ]-→ •\nPush + pull support\n[20 brokers,\n[ remote-write] •\n202 Accepted\n(async) 3×\nreplication]","originalText":"COLLECTION LAYER\n\n[Host Agents] --→\n[Ingestion Gateway\n(50 nodes)] --→\n[Kafka]\n[k8s Exporters]-→ •\nAuth + rate limit\n[Custom SDKs] --→ •\n10M points/sec\ncapacity\n[Prometheus ]-→ •\nPush + pull support\n[20 brokers,\n[ remote-write] •\n202 Accepted\n(async) 3×\nreplication]","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":919.6000000000001,"width":288,"height":207.6,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":937.6000000000001,"width":248,"height":171.6,"fontSize":22,"fontFamily":1,"text":"Alert Router\n\n• Match point\nto rules\n• Fan-out to\nevaluators","originalText":"Alert Router\n\n• Match point\nto rules\n• Fan-out to\nevaluators","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":1171.2,"width":288,"height":236.20000000000002,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":1189.2,"width":248,"height":200.20000000000002,"fontSize":22,"fontFamily":1,"text":"Rollup Worker\n\n• Read raw\nblocks past\nretention\n• Compute agg\n• Write rollup","originalText":"Rollup Worker\n\n• Read raw\nblocks past\nretention\n• Compute agg\n• Write rollup","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":1451.4,"width":288,"height":979.8000000000001,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":1469.4,"width":248,"height":943.8000000000001,"fontSize":22,"fontFamily":1,"text":"STORAGE LAYER\n\n• Time-Series DB\n(TSDB) (100 storage\nnodes)\n• Sharded by\nseries_id\n• WAL + in-memory\nactive\n• 2-hour immutable\nblocks\n• Gorilla compression\n• Raw: 15d, 1m: 3mo,\n1h: 2yr, 1d: forever\n• 2 replicas per\nshard\n\n• Tag Inverted\nIndex (20 nodes)\n• Roaring Bitmaps\n• tag=value →\nseries_ids\n• Bitmap AND/OR for\nqueries\n• ~200 GB total\n\n• Alert State (Redis\nCluster)\n• Sliding windows per\nrule\n• Alert status per\nrule\n• Notification dedup","originalText":"STORAGE LAYER\n\n• Time-Series DB\n(TSDB) (100 storage\nnodes)\n• Sharded by\nseries_id\n• WAL + in-memory\nactive\n• 2-hour immutable\nblocks\n• Gorilla compression\n• Raw: 15d, 1m: 3mo,\n1h: 2yr, 1d: forever\n• 2 replicas per\nshard\n\n• Tag Inverted\nIndex (20 nodes)\n• Roaring Bitmaps\n• tag=value →\nseries_ids\n• Bitmap AND/OR for\nqueries\n• ~200 GB total\n\n• Alert State (Redis\nCluster)\n• Sliding windows per\nrule\n• Alert status per\nrule\n• Notification dedup","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":2475.2000000000003,"width":288,"height":1122.8,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":2493.2000000000003,"width":248,"height":1086.8,"fontSize":22,"fontFamily":1,"text":"QUERY & ALERT\nLAYER\n\n• Query Engine (50\nnodes)\n• Parse query\nlanguage\n• Resolve tags →\nseries\n• Auto-select rollup\ntier\n• Scatter to TSDB\nnodes\n• Gather +\naggregate\n• < 500 ms response\n\n• Alert Evaluators\n(50 nodes)\n• 5M rules, ~83K\nevals/sec\n• Threshold, anomaly,\nSLO\n• State machine per\nrule\nOK → WARN → ALERT\n→ RECOV\nNotification Router\n• PagerDuty, Slack,\nemail\n• Dedup + rate limit\n• Escalation policies\n\n• Notification Router\n• PagerDuty, Slack,\nemail\n• Dedup + rate limit\n• Escalation policies","originalText":"QUERY & ALERT\nLAYER\n\n• Query Engine (50\nnodes)\n• Parse query\nlanguage\n• Resolve tags →\nseries\n• Auto-select rollup\ntier\n• Scatter to TSDB\nnodes\n• Gather +\naggregate\n• < 500 ms response\n\n• Alert Evaluators\n(50 nodes)\n• 5M rules, ~83K\nevals/sec\n• Threshold, anomaly,\nSLO\n• State machine per\nrule\nOK → WARN → ALERT\n→ RECOV\nNotification Router\n• PagerDuty, Slack,\nemail\n• Dedup + rate limit\n• Escalation policies\n\n• Notification Router\n• PagerDuty, Slack,\nemail\n• Dedup + rate limit\n• Escalation policies","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":16,"y":3642,"width":288,"height":522.2,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":36,"y":3660,"width":248,"height":486.2,"fontSize":22,"fontFamily":1,"text":"PRESENTATION\nLAYER\n\n• Dashboard Service\n• Widget rendering\n• Auto-refresh\n(10-60s)\n• Template variables\n• Sharing +\npermissions\n\n• API (Public)\n• Submit metrics\n• Query metrics\n• CRUD\nmonitors/dashboards\n• SLO management","originalText":"PRESENTATION\nLAYER\n\n• Dashboard Service\n• Widget rendering\n• Auto-refresh\n(10-60s)\n• Template variables\n• Sharing +\npermissions\n\n• API (Public)\n• Submit metrics\n• Query metrics\n• CRUD\nmonitors/dashboards\n• SLO management","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":16,"y":4208.2,"width":288,"height":1466,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":36,"y":4226.2,"width":248,"height":1430,"fontSize":22,"fontFamily":1,"text":"Key Self-Metrics:\n• Ingestion rate\n(points/sec, by\ntenant)\n• Ingestion lag\n(Kafka consumer\noffset lag)\n• TSDB write latency\n(p50, p99)\n• Query latency (p50,\np99) — target <\n500ms\n• Active series count\n(total, by tenant)\n• Alert evaluation\nlag (time from data\n→ evaluation)\n• Alert notification\ndelivery latency\n• Storage utilization\nper TSDB node\n• Compression ratio\n(actual vs\ntheoretical)\n• Rollup pipeline lag\n\nAlerts (on the\nmonitoring system\nitself):\nRED — Ingestion lag\n> 5 min (data is\nstale)\nRED — Alert\nevaluation lag > 2\nmin (alerts are\ndelayed)\nRED — TSDB node\nunreachable\nRED — Dead-man's\nswitch not received\n(meta-monitoring\nfailure)\nWARN — Query p99 >\n2 seconds\nWARN — Disk\nutilization > 80% on\nany TSDB node\nWARN — Per-tenant\nseries count\napproaching quota","originalText":"Key Self-Metrics:\n• Ingestion rate\n(points/sec, by\ntenant)\n• Ingestion lag\n(Kafka consumer\noffset lag)\n• TSDB write latency\n(p50, p99)\n• Query latency (p50,\np99) — target <\n500ms\n• Active series count\n(total, by tenant)\n• Alert evaluation\nlag (time from data\n→ evaluation)\n• Alert notification\ndelivery latency\n• Storage utilization\nper TSDB node\n• Compression ratio\n(actual vs\ntheoretical)\n• Rollup pipeline lag\n\nAlerts (on the\nmonitoring system\nitself):\nRED — Ingestion lag\n> 5 min (data is\nstale)\nRED — Alert\nevaluation lag > 2\nmin (alerts are\ndelayed)\nRED — TSDB node\nunreachable\nRED — Dead-man's\nswitch not received\n(meta-monitoring\nfailure)\nWARN — Query p99 >\n2 seconds\nWARN — Disk\nutilization > 80% on\nany TSDB node\nWARN — Per-tenant\nseries count\napproaching quota","textAlign":"left","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":16,"y":5718.2,"width":288,"height":93.2,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":36,"y":5736.2,"width":248,"height":57.2,"fontSize":22,"fontFamily":1,"text":"[Kafka: metrics-raw\ntopic]","originalText":"[Kafka: metrics-raw\ntopic]","textAlign":"left","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":16,"y":5855.4,"width":288,"height":121.8,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":36,"y":5873.4,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Monitoring the\nMonitoring System\n(Meta-Monitoring):","originalText":"Monitoring the\nMonitoring System\n(Meta-Monitoring):","textAlign":"left","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}