{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":121.8,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":34,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"GPU 1\nLlama\n70B","originalText":"GPU 1\nLlama\n70B","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":181.8,"width":288,"height":693.8000000000001,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":199.8,"width":248,"height":657.8000000000001,"fontSize":22,"fontFamily":1,"text":"The GPU Scheduling\nProblem\n\nUnlike CPU\nworkloads, GPU\ninference has unique\nconstraints:\n• GPU memory is\nfixed (80 GB for\nH100) and shared\n• Model weights\nconsume most of\nGPU memory\n• KV cache consumes\nremaining memory (per\nactive request)\n• Loading a model\ntakes 30-120\nseconds (cold start)\n• GPUs cost\n$3-30/hr — idle\ntime is extremely\nexpensive","originalText":"The GPU Scheduling\nProblem\n\nUnlike CPU\nworkloads, GPU\ninference has unique\nconstraints:\n• GPU memory is\nfixed (80 GB for\nH100) and shared\n• Model weights\nconsume most of\nGPU memory\n• KV cache consumes\nremaining memory (per\nactive request)\n• Loading a model\ntakes 30-120\nseconds (cold start)\n• GPUs cost\n$3-30/hr — idle\ntime is extremely\nexpensive","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":919.6000000000001,"width":288,"height":121.8,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":937.6000000000001,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"GPU 2\nMistral\n7B","originalText":"GPU 2\nMistral\n7B","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":1085.4,"width":288,"height":121.8,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":1103.4,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"GPU 3\nSDXL\n(img)","originalText":"GPU 3\nSDXL\n(img)","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":1251.2,"width":288,"height":121.8,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":1269.2,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"GPU 4\nLlama\n70B","originalText":"GPU 4\nLlama\n70B","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":1417,"width":288,"height":207.6,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":1435,"width":248,"height":171.6,"fontSize":22,"fontFamily":1,"text":"GPU 1\nLoaded:\n• Mist\n• Phi-\n• BERT\nFree:","originalText":"GPU 1\nLoaded:\n• Mist\n• Phi-\n• BERT\nFree:","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":16,"y":1668.6,"width":288,"height":1294.4,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":36,"y":1686.6,"width":248,"height":1258.4,"fontSize":22,"fontFamily":1,"text":"Tiered GPU Pool\n\nTier 1: Hot (model\nloaded, GPU running,\nready to serve)\n→ For high-traffic\nmodels: always-on\nreplicas\n→ Serves requests in\n< 100 ms\n\nTier 2: Warm (GPU\nallocated, model\nweights in GPU\nmemory)\n→ For medium-traffic:\nkept loaded but no\nactive requests\n→ Can serve within\n10-50 ms (no cold\nstart)\n→ Costs: GPU\nreserved but\nunderutilized\n\nTier 3: Cold (model\non disk/S3, no GPU\nallocated)\n→ For low-traffic/idle\nmodels: scaled to\nzero\n→ Cold start:\n30-120s (download +\nload model)\n→ Zero cost when not\nin use\n\nOptimization:\nPre-warm based on\npredictive patterns\n\"This model gets\ntraffic at 9am\nweekdays → pre-warm\nat 8:55\"","originalText":"Tiered GPU Pool\n\nTier 1: Hot (model\nloaded, GPU running,\nready to serve)\n→ For high-traffic\nmodels: always-on\nreplicas\n→ Serves requests in\n< 100 ms\n\nTier 2: Warm (GPU\nallocated, model\nweights in GPU\nmemory)\n→ For medium-traffic:\nkept loaded but no\nactive requests\n→ Can serve within\n10-50 ms (no cold\nstart)\n→ Costs: GPU\nreserved but\nunderutilized\n\nTier 3: Cold (model\non disk/S3, no GPU\nallocated)\n→ For low-traffic/idle\nmodels: scaled to\nzero\n→ Cold start:\n30-120s (download +\nload model)\n→ Zero cost when not\nin use\n\nOptimization:\nPre-warm based on\npredictive patterns\n\"This model gets\ntraffic at 9am\nweekdays → pre-warm\nat 8:55\"","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":16,"y":3007,"width":288,"height":121.8,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":36,"y":3025,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Option A: Dedicated\nGPU per Model\n(simple)","originalText":"Option A: Dedicated\nGPU per Model\n(simple)","textAlign":"left","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":16,"y":3172.8,"width":288,"height":264.8,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":36,"y":3190.8,"width":248,"height":228.8,"fontSize":22,"fontFamily":1,"text":"Pros: Simple, no model\nswapping overhead\nCons: Low utilization\n(GPU sits idle if\nmodel isn't used)\nExpensive for 10K\nmodels (can't have\n10K dedicated GPUs)","originalText":"Pros: Simple, no model\nswapping overhead\nCons: Low utilization\n(GPU sits idle if\nmodel isn't used)\nExpensive for 10K\nmodels (can't have\n10K dedicated GPUs)","textAlign":"left","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":16,"y":3481.6000000000004,"width":288,"height":121.8,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":36,"y":3499.6000000000004,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Option B: Model\nMultiplexing (multiple\nmodels per GPU)","originalText":"Option B: Model\nMultiplexing (multiple\nmodels per GPU)","textAlign":"left","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-21"}],"updated":0,"link":null,"locked":false,"id":"shape-20","type":"rectangle","seed":1501,"x":16,"y":3647.4000000000005,"width":288,"height":121.8,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-21","type":"text","seed":1574,"x":36,"y":3665.4000000000005,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"B (14 GB)\nGB)\nB)","originalText":"B (14 GB)\nGB)\nB)","textAlign":"left","verticalAlign":"middle","containerId":"shape-20","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-23"}],"updated":0,"link":null,"locked":false,"id":"shape-22","type":"rectangle","seed":1647,"x":16,"y":3813.2000000000007,"width":288,"height":350.6,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-23","type":"text","seed":1720,"x":36,"y":3831.2000000000007,"width":248,"height":314.6,"fontSize":22,"fontFamily":1,"text":"Pros: Much better\nutilization\nCons: Only works for\nsmall models (7B × 3\n= 21 GB fits in 80\nGB)\nNot applicable for\n70B+ models\nScheduling complexity\n(which models to\nco-locate?)","originalText":"Pros: Much better\nutilization\nCons: Only works for\nsmall models (7B × 3\n= 21 GB fits in 80\nGB)\nNot applicable for\n70B+ models\nScheduling complexity\n(which models to\nco-locate?)","textAlign":"left","verticalAlign":"middle","containerId":"shape-22","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-25"}],"updated":0,"link":null,"locked":false,"id":"shape-24","type":"rectangle","seed":1793,"x":16,"y":4207.800000000001,"width":288,"height":150.4,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-25","type":"text","seed":1866,"x":36,"y":4225.800000000001,"width":248,"height":114.4,"fontSize":22,"fontFamily":1,"text":"Option C:\nScale-to-Zero with\nWarm Pools\n(Recommended)","originalText":"Option C:\nScale-to-Zero with\nWarm Pools\n(Recommended)","textAlign":"left","verticalAlign":"middle","containerId":"shape-24","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":13}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}