{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":30,"y":30,"width":420,"height":68,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":50,"y":52.3,"width":380,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"INFERENCE ENGINE (Per GPU Worker)","originalText":"INFERENCE ENGINE (Per GPU Worker)","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":30,"y":123,"width":544.79956,"height":68,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":50,"y":145.3,"width":504.79956000000004,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"LLM Inference Engine (vLLM / TensorRT-LLM / TGI)","originalText":"LLM Inference Engine (vLLM / TensorRT-LLM / TGI)","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":30,"y":216,"width":420,"height":68,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":50,"y":238.3,"width":380,"height":23.400000000000002,"fontSize":18,"fontFamily":1,"text":"Key techniques:","originalText":"Key techniques:","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":30,"y":309,"width":573.1906,"height":153,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":50,"y":327,"width":533.1906,"height":117,"fontSize":18,"fontFamily":1,"text":"1. Continuous Batching (Orca)\nTraditional: wait for batch to fill → process → return\nContinuous: as requests finish, immediately add new ones\n→ GPU never idle between batches\n→ 2-3× throughput improvement","originalText":"1. Continuous Batching (Orca)\nTraditional: wait for batch to fill → process → return\nContinuous: as requests finish, immediately add new ones\n→ GPU never idle between batches\n→ 2-3× throughput improvement","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":30,"y":487,"width":507.91792000000004,"height":153,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":50,"y":505,"width":467.91792000000004,"height":117,"fontSize":18,"fontFamily":1,"text":"2. PagedAttention (vLLM)\nKV cache managed like virtual memory pages\n→ No pre-allocated contiguous memory per request\n→ 2-4× more concurrent requests per GPU\n→ Near-zero memory waste from fragmentation","originalText":"2. PagedAttention (vLLM)\nKV cache managed like virtual memory pages\n→ No pre-allocated contiguous memory per request\n→ 2-4× more concurrent requests per GPU\n→ Near-zero memory waste from fragmentation","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":30,"y":665,"width":561.58996,"height":153,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":50,"y":683,"width":521.58996,"height":117,"fontSize":18,"fontFamily":1,"text":"3. Tensor Parallelism\nModel too large for one GPU (70B = ~140 GB in fp16)\n→ Split across 4 GPUs: each holds 1/4 of each layer\n→ All-reduce after each layer (NVLink: 900 GB/s)\n→ Appears as single model to the caller","originalText":"3. Tensor Parallelism\nModel too large for one GPU (70B = ~140 GB in fp16)\n→ Split across 4 GPUs: each holds 1/4 of each layer\n→ All-reduce after each layer (NVLink: 900 GB/s)\n→ Appears as single model to the caller","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":30,"y":843,"width":589.5994000000001,"height":129.60000000000002,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":50,"y":861,"width":549.5994000000001,"height":93.60000000000001,"fontSize":18,"fontFamily":1,"text":"4. Speculative Decoding\nUse small \"draft\" model to predict N tokens ahead\nLarge model verifies in one forward pass (parallel verify)\nAccept all correct predictions → 2-3× speed for easy text","originalText":"4. Speculative Decoding\nUse small \"draft\" model to predict N tokens ahead\nLarge model verifies in one forward pass (parallel verify)\nAccept all correct predictions → 2-3× speed for easy text","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}