{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":93.2,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":34,"width":248,"height":57.2,"fontSize":22,"fontFamily":1,"text":"INFERENCE ENGINE\n(Per GPU Worker)","originalText":"INFERENCE ENGINE\n(Per GPU Worker)","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":153.2,"width":288,"height":150.4,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":171.2,"width":248,"height":114.4,"fontSize":22,"fontFamily":1,"text":"LLM Inference Engine\n(vLLM /\nTensorRT-LLM /\nTGI)","originalText":"LLM Inference Engine\n(vLLM /\nTensorRT-LLM /\nTGI)","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":347.6,"width":288,"height":68,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":367.3,"width":248,"height":28.6,"fontSize":22,"fontFamily":1,"text":"Key techniques:","originalText":"Key techniques:","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":459.6,"width":288,"height":407.8,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":477.6,"width":248,"height":371.8,"fontSize":22,"fontFamily":1,"text":"1. Continuous\nBatching (Orca)\nTraditional: wait for\nbatch to fill →\nprocess → return\nContinuous: as\nrequests finish,\nimmediately add new\nones\n→ GPU never idle\nbetween batches\n→ 2-3× throughput\nimprovement","originalText":"1. Continuous\nBatching (Orca)\nTraditional: wait for\nbatch to fill →\nprocess → return\nContinuous: as\nrequests finish,\nimmediately add new\nones\n→ GPU never idle\nbetween batches\n→ 2-3× throughput\nimprovement","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":911.4000000000001,"width":288,"height":436.40000000000003,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":929.4000000000001,"width":248,"height":400.40000000000003,"fontSize":22,"fontFamily":1,"text":"2. PagedAttention\n(vLLM)\nKV cache managed\nlike virtual memory\npages\n→ No pre-allocated\ncontiguous memory\nper request\n→ 2-4× more\nconcurrent requests\nper GPU\n→ Near-zero memory\nwaste from\nfragmentation","originalText":"2. PagedAttention\n(vLLM)\nKV cache managed\nlike virtual memory\npages\n→ No pre-allocated\ncontiguous memory\nper request\n→ 2-4× more\nconcurrent requests\nper GPU\n→ Near-zero memory\nwaste from\nfragmentation","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":1391.8000000000002,"width":288,"height":379.2,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":1409.8000000000002,"width":248,"height":343.2,"fontSize":22,"fontFamily":1,"text":"3. Tensor Parallelism\nModel too large for\none GPU (70B =\n~140 GB in fp16)\n→ Split across 4\nGPUs: each holds 1/4\nof each layer\n→ All-reduce after\neach layer (NVLink:\n900 GB/s)\n→ Appears as single\nmodel to the caller","originalText":"3. Tensor Parallelism\nModel too large for\none GPU (70B =\n~140 GB in fp16)\n→ Split across 4\nGPUs: each holds 1/4\nof each layer\n→ All-reduce after\neach layer (NVLink:\n900 GB/s)\n→ Appears as single\nmodel to the caller","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":16,"y":1815.0000000000002,"width":288,"height":350.6,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":36,"y":1833.0000000000002,"width":248,"height":314.6,"fontSize":22,"fontFamily":1,"text":"4. Speculative\nDecoding\nUse small \"draft\"\nmodel to predict N\ntokens ahead\nLarge model verifies\nin one forward pass\n(parallel verify)\nAccept all correct\npredictions → 2-3×\nspeed for easy text","originalText":"4. Speculative\nDecoding\nUse small \"draft\"\nmodel to predict N\ntokens ahead\nLarge model verifies\nin one forward pass\n(parallel verify)\nAccept all correct\npredictions → 2-3×\nspeed for easy text","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}