{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":93.2,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":34,"width":248,"height":57.2,"fontSize":22,"fontFamily":1,"text":"KV Cache Sharing for\nCommon Prefixes","originalText":"KV Cache Sharing for\nCommon Prefixes","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":153.2,"width":288,"height":121.8,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":171.2,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Problem: Many\nrequests share the\nsame system prompt","originalText":"Problem: Many\nrequests share the\nsame system prompt","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":319,"width":288,"height":264.8,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":337,"width":248,"height":228.8,"fontSize":22,"fontFamily":1,"text":"Request 1: [System:\nYou are a helpful\nassistant.] + [User:\nHi]\nRequest 2: [System:\nYou are a helpful\nassistant.] + [User:\nBye]","originalText":"Request 1: [System:\nYou are a helpful\nassistant.] + [User:\nHi]\nRequest 2: [System:\nYou are a helpful\nassistant.] + [User:\nBye]","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":627.8,"width":288,"height":264.8,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":645.8,"width":248,"height":228.8,"fontSize":22,"fontFamily":1,"text":"Naive: Each request\ncomputes KV cache\nfor the system\nprompt\n→ Wasted: system\nprompt processed\n20K times/sec =\npure waste","originalText":"Naive: Each request\ncomputes KV cache\nfor the system\nprompt\n→ Wasted: system\nprompt processed\n20K times/sec =\npure waste","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":936.5999999999999,"width":288,"height":436.40000000000003,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":954.5999999999999,"width":248,"height":400.40000000000003,"fontSize":22,"fontFamily":1,"text":"Prefix caching:\n1. Compute KV cache\nfor \"You are a\nhelpful assistant.\"\nonce\n2. Cache it:\nhash(system_prompt)\n→ KV cache pages\n3. New requests:\ncopy cached KV\npages → append user\ntokens\n4. Skip prefill for the\nshared prefix entirely","originalText":"Prefix caching:\n1. Compute KV cache\nfor \"You are a\nhelpful assistant.\"\nonce\n2. Cache it:\nhash(system_prompt)\n→ KV cache pages\n3. New requests:\ncopy cached KV\npages → append user\ntokens\n4. Skip prefill for the\nshared prefix entirely","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":1417,"width":288,"height":322,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":1435,"width":248,"height":286,"fontSize":22,"fontFamily":1,"text":"Savings:\nIf system prompt =\n500 tokens, user\nmessage = 50\ntokens:\n→ 90% of prefill\ncomputation\neliminated\n→ First-token latency:\n50ms → 5ms","originalText":"Savings:\nIf system prompt =\n500 tokens, user\nmessage = 50\ntokens:\n→ 90% of prefill\ncomputation\neliminated\n→ First-token latency:\n50ms → 5ms","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":16,"y":1783,"width":288,"height":236.20000000000002,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":36,"y":1801,"width":248,"height":200.20000000000002,"fontSize":22,"fontFamily":1,"text":"Implementation: vLLM\nautomatic prefix\ncaching (APC)\nRadix tree of cached\nprefixes →\nlongest-prefix\nmatch","originalText":"Implementation: vLLM\nautomatic prefix\ncaching (APC)\nRadix tree of cached\nprefixes →\nlongest-prefix\nmatch","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}