{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"},{"type":"arrow","id":"shape-16"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":36,"y":32.00000000000001,"width":192,"height":82.80000000000001,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":56,"y":50.000000000000014,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Client\n(SDK)","originalText":"Client\n(SDK)","textAlign":"center","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"},{"type":"arrow","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":436.03742,"y":32,"width":197.92516,"height":153,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":456.03742,"y":50,"width":157.92516,"height":117,"fontSize":18,"fontFamily":1,"text":"API Gateway\n• API key auth\n• Rate limiting\n• Usage metering\n• Route by model","originalText":"API Gateway\n• API key auth\n• Rate limiting\n• Usage metering\n• Route by model","textAlign":"center","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-16","type":"arrow","seed":1209,"x":132,"y":114.80000000000001,"width":0,"height":82,"points":[[0,0],[0,41],[0,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-0","focus":0,"gap":1},"endBinding":{"elementId":"shape-4","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"},{"type":"arrow","id":"shape-16"},{"type":"arrow","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":36,"y":196.8,"width":192,"height":82.80000000000001,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":56,"y":214.8,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"API\nGateway","originalText":"API\nGateway","textAlign":"center","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"arrow","seed":1282,"x":132,"y":279.6,"width":0,"height":82,"points":[[0,0],[0,41],[0,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-4","focus":0,"gap":1},"endBinding":{"elementId":"shape-6","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"},{"type":"arrow","id":"shape-17"},{"type":"arrow","id":"shape-18"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":36,"y":361.6,"width":192,"height":82.80000000000001,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":56,"y":379.6,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Router /\nScheduler","originalText":"Router /\nScheduler","textAlign":"center","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"arrow","seed":1428,"x":535,"y":185,"width":0,"height":82,"points":[[0,0],[0,41],[0,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-8","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"},{"type":"arrow","id":"shape-19"},{"type":"arrow","id":"shape-20"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":414.5915,"y":267,"width":240.81700000000004,"height":457.2,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":434.5915,"y":285,"width":200.81700000000004,"height":421.2,"fontSize":18,"fontFamily":1,"text":"INFERENCE ROUTER\n\n• Request Router\n• Model → endpoint\n• Version routing\n• Canary split\n\n• Queue Manager\n• Request queue\nper model\n• Priority queues\n• Overflow handling\n\n• Load Balancer\n• Least-loaded\nGPU selection\n• Batch formation\n• KV cache aware","originalText":"INFERENCE ROUTER\n\n• Request Router\n• Model → endpoint\n• Version routing\n• Canary split\n\n• Queue Manager\n• Request queue\nper model\n• Priority queues\n• Overflow handling\n\n• Load Balancer\n• Least-loaded\nGPU selection\n• Batch formation\n• KV cache aware","textAlign":"center","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-18","type":"arrow","seed":1355,"x":132,"y":444.4,"width":0,"height":82.00000000000011,"points":[[0,0],[0,41.00000000000006],[0,82.00000000000011]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-6","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"},{"type":"arrow","id":"shape-18"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":36,"y":526.4000000000001,"width":192,"height":82.80000000000001,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":56,"y":544.4000000000001,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"GPU\nWorkers","originalText":"GPU\nWorkers","textAlign":"center","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-20","type":"arrow","seed":1501,"x":535,"y":724.2,"width":0,"height":82.00000000000011,"points":[[0,0],[0,41],[0,82.00000000000011]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-8","focus":0,"gap":1},"endBinding":{"elementId":"shape-12","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"},{"type":"arrow","id":"shape-20"},{"type":"arrow","id":"shape-21"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":340,"y":806.2000000000002,"width":390,"height":457.2,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":360,"y":824.2000000000002,"width":350,"height":421.2,"fontSize":18,"fontFamily":1,"text":"GPU WORKER FLEET\n\n• LLM Workers (H100 × 4-8 per pod)\n• vLLM / TGI engine\n• KV cache management\n• Continuous batching\n• Tensor parallelism\n\n• Small Model Workers (L4,\nmulti-model)\n• Triton Server\n• Model multiplexing\n• GPU time-sharing\n\n• Batch Workers (A100 elastic)\n• High throughput\n• Not latency-\nsensitive","originalText":"GPU WORKER FLEET\n\n• LLM Workers (H100 × 4-8 per pod)\n• vLLM / TGI engine\n• KV cache management\n• Continuous batching\n• Tensor parallelism\n\n• Small Model Workers (L4,\nmulti-model)\n• Triton Server\n• Model multiplexing\n• GPU time-sharing\n\n• Batch Workers (A100 elastic)\n• High throughput\n• Not latency-\nsensitive","textAlign":"center","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-21","type":"arrow","seed":1574,"x":535,"y":1263.4,"width":0,"height":82,"points":[[0,0],[0,41],[0,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-12","focus":0,"gap":1},"endBinding":{"elementId":"shape-14","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"},{"type":"arrow","id":"shape-21"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":410.89952,"y":1345.4,"width":248.20096000000004,"height":457.2,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":430.89952,"y":1363.4,"width":208.20096000000004,"height":421.2,"fontSize":18,"fontFamily":1,"text":"PLATFORM SERVICES\n\n• Model Registry\n• Artifact storage\n• Version control\n• Model metadata\n\n• Autoscaler\n• GPU provisioning\n• Scale-to-zero\n• Predictive\nscaling\n\n• Metering /\nBilling\n• Token counting\n• Per-customer\nusage tracking","originalText":"PLATFORM SERVICES\n\n• Model Registry\n• Artifact storage\n• Version control\n• Model metadata\n\n• Autoscaler\n• GPU provisioning\n• Scale-to-zero\n• Predictive\nscaling\n\n• Metering /\nBilling\n• Token counting\n• Per-customer\nusage tracking","textAlign":"center","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-23"}],"updated":0,"link":null,"locked":false,"id":"shape-22","type":"rectangle","seed":1647,"x":36,"y":1842.6000000000001,"width":420,"height":80.2,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-23","type":"text","seed":1720,"x":56,"y":1860.6000000000001,"width":380,"height":44.2,"fontSize":17,"fontFamily":1,"text":"Model Registry Autoscaler\n(S3 + DB) (GPU Orchestrator)","originalText":"Model Registry Autoscaler\n(S3 + DB) (GPU Orchestrator)","textAlign":"left","verticalAlign":"middle","containerId":"shape-22","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-25"}],"updated":0,"link":null,"locked":false,"id":"shape-24","type":"rectangle","seed":1793,"x":36,"y":1946.8000000000002,"width":420,"height":68,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-25","type":"text","seed":1866,"x":56,"y":1969.7500000000002,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Detailed:","originalText":"Detailed:","textAlign":"left","verticalAlign":"middle","containerId":"shape-24","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-27"}],"updated":0,"link":null,"locked":false,"id":"shape-26","type":"rectangle","seed":1939,"x":36,"y":2038.8000000000002,"width":420,"height":68,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-27","type":"text","seed":2012,"x":56,"y":2061.75,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Client (API / SDK)","originalText":"Client (API / SDK)","textAlign":"left","verticalAlign":"middle","containerId":"shape-26","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":11}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}