{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"},{"type":"arrow","id":"shape-20"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":244.0983999999999,"y":32,"width":294.45088,"height":106.2,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":264.0983999999999,"y":50,"width":254.45087999999998,"height":70.2,"fontSize":18,"fontFamily":1,"text":"CDN / Edge\n(API docs, SDK\ndownloads)","originalText":"CDN / Edge\n(API docs, SDK\ndownloads)","textAlign":"center","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-20","type":"arrow","seed":1501,"x":391.3238399999999,"y":138.2,"width":0,"height":82,"points":[[0,0],[0,41],[0,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-0","focus":0,"gap":1},"endBinding":{"elementId":"shape-2","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"},{"type":"arrow","id":"shape-20"},{"type":"arrow","id":"shape-21"},{"type":"arrow","id":"shape-22"},{"type":"arrow","id":"shape-23"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":269.2172199999999,"y":220.2,"width":244.21324,"height":68,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":289.2172199999999,"y":242.5,"width":204.21324,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"Global Load Balancer","originalText":"Global Load Balancer","textAlign":"center","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-23","type":"arrow","seed":1720,"x":275.76842986666657,"y":288.2,"width":139.3462298666666,"height":112.80000000000001,"points":[[0,0],[-139.3462298666666,41],[-139.3462298666666,112.80000000000001]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-4","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"},{"type":"arrow","id":"shape-23"},{"type":"arrow","id":"shape-25"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":35.99999999999997,"y":401,"width":200.8444,"height":68,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":55.99999999999997,"y":423.3,"width":160.8444,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"API Gateway (c)","originalText":"API Gateway (c)","textAlign":"center","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-22","type":"arrow","seed":1647,"x":391.3238399999999,"y":288.2,"width":0,"height":112.80000000000001,"points":[[0,0],[0,41],[0,112.80000000000001]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-6","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"},{"type":"arrow","id":"shape-22"},{"type":"arrow","id":"shape-24"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":290.8443999999999,"y":401,"width":200.95888000000002,"height":68,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":310.8443999999999,"y":423.3,"width":160.95888000000002,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"API Gateway (b)","originalText":"API Gateway (b)","textAlign":"center","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-21","type":"arrow","seed":1574,"x":507.5928421333332,"y":288.2,"width":140.20673786666669,"height":82,"points":[[0,0],[140.20673786666669,41],[140.20673786666669,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-8","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"},{"type":"arrow","id":"shape-21"},{"type":"arrow","id":"shape-26"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":545.8032799999999,"y":370.2,"width":203.9926,"height":129.60000000000002,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":565.8032799999999,"y":388.2,"width":163.9926,"height":93.60000000000001,"fontSize":18,"fontFamily":1,"text":"API Gateway (a)\n• API key auth\n• Rate limiting\n• Usage metering","originalText":"API Gateway (a)\n• API key auth\n• Rate limiting\n• Usage metering","textAlign":"center","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-26","type":"arrow","seed":1939,"x":647.7995799999999,"y":499.8,"width":136.38205999999997,"height":97.25959768358592,"points":[[0,0],[0,40.99999999999994],[-136.38205999999997,97.25959768358592]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-8","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-25","type":"arrow","seed":1866,"x":136.42219999999998,"y":469,"width":134.80795999999992,"height":127.75366969000265,"points":[[0,0],[0,71.79999999999995],[134.80795999999992,127.75366969000265]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-4","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-24","type":"arrow","seed":1793,"x":391.3238399999999,"y":469,"width":0,"height":112.79999999999995,"points":[[0,0],[0,71.79999999999995],[0,112.79999999999995]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-6","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"},{"type":"arrow","id":"shape-24"},{"type":"arrow","id":"shape-25"},{"type":"arrow","id":"shape-26"},{"type":"arrow","id":"shape-27"},{"type":"arrow","id":"shape-28"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":271.2301599999999,"y":581.8,"width":240.18736000000004,"height":129.60000000000002,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":291.2301599999999,"y":599.8,"width":200.18736000000004,"height":93.60000000000001,"fontSize":18,"fontFamily":1,"text":"Inference Router\n(model → endpoint\nrouting, load balance,\nqueue management)","originalText":"Inference Router\n(model → endpoint\nrouting, load balance,\nqueue management)","textAlign":"center","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-28","type":"arrow","seed":2085,"x":306.958600831758,"y":711.3999999999999,"width":53.37924083175807,"height":82.00000000000011,"points":[[0,0],[-53.37924083175807,41.000000000000114],[-53.37924083175807,82.00000000000011]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-10","focus":0,"gap":1},"endBinding":{"elementId":"shape-12","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"},{"type":"arrow","id":"shape-28"},{"type":"arrow","id":"shape-29"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":145.98307999999994,"y":793.4,"width":215.19256000000001,"height":387,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":165.98307999999994,"y":811.4,"width":175.19256000000001,"height":351,"fontSize":18,"fontFamily":1,"text":"Small Model Pool\n\nL4 / A10G\nTriton Inference\nServer\n\n• Multi-model\n• GPU time-sharing\n• Sub-10ms latency\n\nModels:\n• Embeddings\n• Classifiers\n• Rerankers\n• Sentiment","originalText":"Small Model Pool\n\nL4 / A10G\nTriton Inference\nServer\n\n• Multi-model\n• GPU time-sharing\n• Sub-10ms latency\n\nModels:\n• Embeddings\n• Classifiers\n• Rerankers\n• Sentiment","textAlign":"center","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-27","type":"arrow","seed":2012,"x":475.68907916824185,"y":711.3999999999999,"width":53.379240831758125,"height":82.00000000000011,"points":[[0,0],[53.379240831758125,41.000000000000114],[53.379240831758125,82.00000000000011]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-10","focus":0,"gap":1},"endBinding":{"elementId":"shape-14","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"},{"type":"arrow","id":"shape-27"},{"type":"arrow","id":"shape-30"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":415.17563999999993,"y":793.4,"width":227.78536000000003,"height":387,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":435.17563999999993,"y":811.4,"width":187.78536000000003,"height":351,"fontSize":18,"fontFamily":1,"text":"LLM GPU Cluster\n\nH100 × 4-8/pod\nvLLM engine\n\n• Continuous batch\n• PagedAttention\n• Tensor parallel\n• Prefix caching\n• Multi-LoRA\n\nModels:\n• Llama 70B/405B\n• Mixtral 8×22B\n• Claude/GPT proxy","originalText":"LLM GPU Cluster\n\nH100 × 4-8/pod\nvLLM engine\n\n• Continuous batch\n• PagedAttention\n• Tensor parallel\n• Prefix caching\n• Multi-LoRA\n\nModels:\n• Llama 70B/405B\n• Mixtral 8×22B\n• Claude/GPT proxy","textAlign":"center","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-30","type":"arrow","seed":2231,"x":529.06832,"y":1180.4,"width":11.760774010828868,"height":82,"points":[[0,0],[0,41],[-11.760774010828868,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-14","focus":0,"gap":1},"endBinding":{"elementId":"shape-16","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-29","type":"arrow","seed":2158,"x":253.57935999999995,"y":1180.4,"width":11.760774010828868,"height":82,"points":[[0,0],[0,41],[11.760774010828868,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-12","focus":0,"gap":1},"endBinding":{"elementId":"shape-16","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"},{"type":"arrow","id":"shape-29"},{"type":"arrow","id":"shape-30"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":196.3238399999999,"y":1262.4,"width":390,"height":878.4,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":216.3238399999999,"y":1280.4,"width":350,"height":842.4,"fontSize":18,"fontFamily":1,"text":"PLATFORM LAYER\n\n• Model Registry (S3 + Postgres)\n• Artifacts\n• Versions\n• Configs\n• LoRA adapters\n\n• GPU Orchestrator (Kubernetes +\ncustom GPU scheduler)\n• Provision/\ndecommission\n• Scale-to-zero\n• Bin-packing\n• Spot reclaim\n\n• Metering + Billing (Kafka →\nClickHouse → Stripe integration)\n• Token-level tracking\n• Per-customer billing\n• Rate limit state\n• Usage dashboards\n\n• Autoscaler\n• Multi-signal\n(queue, GPU,\nlatency)\n• Predictive\n• Scale-to-zero\n\n• Monitoring\n• GPU util/temp\n• Latency p50/99\n• Throughput\n• Error rates\n• Queue depth","originalText":"PLATFORM LAYER\n\n• Model Registry (S3 + Postgres)\n• Artifacts\n• Versions\n• Configs\n• LoRA adapters\n\n• GPU Orchestrator (Kubernetes +\ncustom GPU scheduler)\n• Provision/\ndecommission\n• Scale-to-zero\n• Bin-packing\n• Spot reclaim\n\n• Metering + Billing (Kafka →\nClickHouse → Stripe integration)\n• Token-level tracking\n• Per-customer billing\n• Rate limit state\n• Usage dashboards\n\n• Autoscaler\n• Multi-signal\n(queue, GPU,\nlatency)\n• Predictive\n• Scale-to-zero\n\n• Monitoring\n• GPU util/temp\n• Latency p50/99\n• Throughput\n• Error rates\n• Queue depth","textAlign":"center","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":36,"y":2254.8,"width":390,"height":784.8000000000001,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":56,"y":2272.8,"width":350,"height":748.8000000000001,"fontSize":18,"fontFamily":1,"text":"Key Metrics:\n• Time to first token (TTFT) —\np50, p99\n• Tokens per second (TPS) per GPU,\nper model\n• GPU utilization (compute %,\nmemory %)\n• KV cache utilization per GPU\n• Queue depth per endpoint\n• Cold start rate and duration\n• Request success/error rate\n• Inter-token latency (ITL) for\nstreaming\n• Cost per 1M tokens (by model, by\ncustomer)\n• GPU idle time (cost waste metric)\n\nAlerts:\nRED — GPU OOM events (admission\ncontrol failure)\nRED — Error rate > 5% on any\nendpoint\nRED — TTFT p99 > 5 seconds\n(degraded experience)\nRED — Queue depth growing for > 5\nminutes (autoscaler stuck)\nWARN — GPU utilization < 30%\n(wasting money)\nWARN — Cold start rate > 10% of\nrequests\nWARN — Model load failure on any\nGPU node","originalText":"Key Metrics:\n• Time to first token (TTFT) —\np50, p99\n• Tokens per second (TPS) per GPU,\nper model\n• GPU utilization (compute %,\nmemory %)\n• KV cache utilization per GPU\n• Queue depth per endpoint\n• Cold start rate and duration\n• Request success/error rate\n• Inter-token latency (ITL) for\nstreaming\n• Cost per 1M tokens (by model, by\ncustomer)\n• GPU idle time (cost waste metric)\n\nAlerts:\nRED — GPU OOM events (admission\ncontrol failure)\nRED — Error rate > 5% on any\nendpoint\nRED — TTFT p99 > 5 seconds\n(degraded experience)\nRED — Queue depth growing for > 5\nminutes (autoscaler stuck)\nWARN — GPU utilization < 30%\n(wasting money)\nWARN — Cold start rate > 10% of\nrequests\nWARN — Model load failure on any\nGPU node","textAlign":"center","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-32"}],"updated":0,"link":null,"locked":false,"id":"shape-31","type":"rectangle","seed":2304,"x":36,"y":3079.6000000000004,"width":420,"height":102.3,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-32","type":"text","seed":2377,"x":56,"y":3097.6000000000004,"width":380,"height":66.3,"fontSize":17,"fontFamily":1,"text":"Batch Processing\nPool\nA100 (spot/preempt)","originalText":"Batch Processing\nPool\nA100 (spot/preempt)","textAlign":"left","verticalAlign":"middle","containerId":"shape-31","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-34"}],"updated":0,"link":null,"locked":false,"id":"shape-33","type":"rectangle","seed":2450,"x":36,"y":3205.9000000000005,"width":420,"height":80.2,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-34","type":"text","seed":2523,"x":56,"y":3223.9000000000005,"width":380,"height":44.2,"fontSize":17,"fontFamily":1,"text":"High throughput\nNo latency SLA","originalText":"High throughput\nNo latency SLA","textAlign":"left","verticalAlign":"middle","containerId":"shape-33","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-36"}],"updated":0,"link":null,"locked":false,"id":"shape-35","type":"rectangle","seed":2596,"x":36,"y":3310.1000000000004,"width":420,"height":102.3,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-36","type":"text","seed":2669,"x":56,"y":3328.1000000000004,"width":380,"height":66.3,"fontSize":17,"fontFamily":1,"text":"Reads from S3 JSONL\nWrites results\nback to S3","originalText":"Reads from S3 JSONL\nWrites results\nback to S3","textAlign":"left","verticalAlign":"middle","containerId":"shape-35","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-14"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-38"}],"updated":0,"link":null,"locked":false,"id":"shape-37","type":"rectangle","seed":2742,"x":36,"y":3436.4000000000005,"width":420,"height":68,"customData":{"revealStep":14}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-14"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-38","type":"text","seed":2815,"x":56,"y":3459.3500000000004,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Monitoring:","originalText":"Monitoring:","textAlign":"left","verticalAlign":"middle","containerId":"shape-37","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":14}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}