{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":466,"y":32,"width":192,"height":106.2,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":486,"y":50,"width":152,"height":70.2,"fontSize":18,"fontFamily":1,"text":"GPU 1\nLlama\n70B","originalText":"GPU 1\nLlama\n70B","textAlign":"center","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":36,"y":32,"width":390,"height":363.6,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":56,"y":50,"width":350,"height":327.6,"fontSize":18,"fontFamily":1,"text":"The GPU Scheduling Problem\n\nUnlike CPU workloads, GPU inference\nhas unique constraints:\n• GPU memory is fixed (80 GB for\nH100) and shared\n• Model weights consume most of\nGPU memory\n• KV cache consumes remaining\nmemory (per active request)\n• Loading a model takes 30-120\nseconds (cold start)\n• GPUs cost $3-30/hr — idle time\nis extremely expensive","originalText":"The GPU Scheduling Problem\n\nUnlike CPU workloads, GPU inference\nhas unique constraints:\n• GPU memory is fixed (80 GB for\nH100) and shared\n• Model weights consume most of\nGPU memory\n• KV cache consumes remaining\nmemory (per active request)\n• Loading a model takes 30-120\nseconds (cold start)\n• GPUs cost $3-30/hr — idle time\nis extremely expensive","textAlign":"center","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":36,"y":435.5999999999999,"width":192,"height":106.2,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":56,"y":453.5999999999999,"width":152,"height":70.2,"fontSize":18,"fontFamily":1,"text":"GPU 2\nMistral\n7B","originalText":"GPU 2\nMistral\n7B","textAlign":"center","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":268,"y":435.5999999999999,"width":192,"height":106.2,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":288,"y":453.5999999999999,"width":152,"height":70.2,"fontSize":18,"fontFamily":1,"text":"GPU 3\nSDXL\n(img)","originalText":"GPU 3\nSDXL\n(img)","textAlign":"center","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":36,"y":581.8,"width":192,"height":106.2,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":56,"y":599.8,"width":152,"height":70.2,"fontSize":18,"fontFamily":1,"text":"GPU 4\nLlama\n70B","originalText":"GPU 4\nLlama\n70B","textAlign":"center","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":268,"y":581.8,"width":192,"height":176.4,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":288,"y":599.8,"width":152,"height":140.4,"fontSize":18,"fontFamily":1,"text":"GPU 1\nLoaded:\n• Mist\n• Phi-\n• BERT\nFree:","originalText":"GPU 1\nLoaded:\n• Mist\n• Phi-\n• BERT\nFree:","textAlign":"center","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":36,"y":798.2,"width":390,"height":714.6,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":56,"y":816.2,"width":350,"height":678.6,"fontSize":18,"fontFamily":1,"text":"Tiered GPU Pool\n\nTier 1: Hot (model loaded, GPU\nrunning, ready to serve)\n→ For high-traffic models: always-on\nreplicas\n→ Serves requests in < 100 ms\n\nTier 2: Warm (GPU allocated, model\nweights in GPU memory)\n→ For medium-traffic: kept loaded\nbut no active requests\n→ Can serve within 10-50 ms (no cold\nstart)\n→ Costs: GPU reserved but\nunderutilized\n\nTier 3: Cold (model on disk/S3, no\nGPU allocated)\n→ For low-traffic/idle models: scaled\nto zero\n→ Cold start: 30-120s (download +\nload model)\n→ Zero cost when not in use\n\nOptimization: Pre-warm based on\npredictive patterns\n\"This model gets traffic at 9am\nweekdays → pre-warm at 8:55\"","originalText":"Tiered GPU Pool\n\nTier 1: Hot (model loaded, GPU\nrunning, ready to serve)\n→ For high-traffic models: always-on\nreplicas\n→ Serves requests in < 100 ms\n\nTier 2: Warm (GPU allocated, model\nweights in GPU memory)\n→ For medium-traffic: kept loaded\nbut no active requests\n→ Can serve within 10-50 ms (no cold\nstart)\n→ Costs: GPU reserved but\nunderutilized\n\nTier 3: Cold (model on disk/S3, no\nGPU allocated)\n→ For low-traffic/idle models: scaled\nto zero\n→ Cold start: 30-120s (download +\nload model)\n→ Zero cost when not in use\n\nOptimization: Pre-warm based on\npredictive patterns\n\"This model gets traffic at 9am\nweekdays → pre-warm at 8:55\"","textAlign":"center","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":36,"y":1552.8000000000002,"width":430.51142,"height":68,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":56,"y":1575.7500000000002,"width":390.51142,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Option A: Dedicated GPU per Model (simple)","originalText":"Option A: Dedicated GPU per Model (simple)","textAlign":"left","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":36,"y":1644.8000000000002,"width":566.3642,"height":102.3,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":56,"y":1662.8000000000002,"width":526.3642,"height":66.3,"fontSize":17,"fontFamily":1,"text":"Pros: Simple, no model swapping overhead\nCons: Low utilization (GPU sits idle if model isn't used)\nExpensive for 10K models (can't have 10K dedicated GPUs)","originalText":"Pros: Simple, no model swapping overhead\nCons: Low utilization (GPU sits idle if model isn't used)\nExpensive for 10K models (can't have 10K dedicated GPUs)","textAlign":"left","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":36,"y":1771.1000000000001,"width":520.62944,"height":68,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":56,"y":1794.0500000000002,"width":480.62944000000005,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Option B: Model Multiplexing (multiple models per GPU)","originalText":"Option B: Model Multiplexing (multiple models per GPU)","textAlign":"left","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-21"}],"updated":0,"link":null,"locked":false,"id":"shape-20","type":"rectangle","seed":1501,"x":36,"y":1863.1000000000001,"width":420,"height":102.3,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-21","type":"text","seed":1574,"x":56,"y":1881.1000000000001,"width":380,"height":66.3,"fontSize":17,"fontFamily":1,"text":"B (14 GB)\nGB)\nB)","originalText":"B (14 GB)\nGB)\nB)","textAlign":"left","verticalAlign":"middle","containerId":"shape-20","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-23"}],"updated":0,"link":null,"locked":false,"id":"shape-22","type":"rectangle","seed":1647,"x":36,"y":1989.4,"width":637.8135,"height":124.4,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-23","type":"text","seed":1720,"x":56,"y":2007.4,"width":597.8135,"height":88.4,"fontSize":17,"fontFamily":1,"text":"Pros: Much better utilization\nCons: Only works for small models (7B × 3 = 21 GB fits in 80 GB)\nNot applicable for 70B+ models\nScheduling complexity (which models to co-locate?)","originalText":"Pros: Much better utilization\nCons: Only works for small models (7B × 3 = 21 GB fits in 80 GB)\nNot applicable for 70B+ models\nScheduling complexity (which models to co-locate?)","textAlign":"left","verticalAlign":"middle","containerId":"shape-22","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-25"}],"updated":0,"link":null,"locked":false,"id":"shape-24","type":"rectangle","seed":1793,"x":36,"y":2137.8,"width":539.2080599999999,"height":80.2,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-25","type":"text","seed":1866,"x":56,"y":2155.8,"width":499.20805999999993,"height":44.2,"fontSize":17,"fontFamily":1,"text":"Option C: Scale-to-Zero with Warm Pools\n(Recommended)","originalText":"Option C: Scale-to-Zero with Warm Pools\n(Recommended)","textAlign":"left","verticalAlign":"middle","containerId":"shape-24","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":13}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}