{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":93.2,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":34,"width":248,"height":57.2,"fontSize":22,"fontFamily":1,"text":"RAG Evaluation\nFramework","originalText":"RAG Evaluation\nFramework","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":153.2,"width":288,"height":493.6,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":171.2,"width":248,"height":457.6,"fontSize":22,"fontFamily":1,"text":"Dimension 1: Retrieval\nQuality\n• Recall@K: % of\nrelevant chunks in\ntop-K results\n• MRR (Mean\nReciprocal Rank): how\nhigh is the first\nrelevant?\n• NDCG: quality of\nranking considering\ngraded relevance\nMeasured via:\nhuman-labeled test\nset of (query,\nrelevant_chunks)","originalText":"Dimension 1: Retrieval\nQuality\n• Recall@K: % of\nrelevant chunks in\ntop-K results\n• MRR (Mean\nReciprocal Rank): how\nhigh is the first\nrelevant?\n• NDCG: quality of\nranking considering\ngraded relevance\nMeasured via:\nhuman-labeled test\nset of (query,\nrelevant_chunks)","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":690.8,"width":288,"height":522.2,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":708.8,"width":248,"height":486.2,"fontSize":22,"fontFamily":1,"text":"Dimension 2:\nGeneration Quality\n• Faithfulness: are\nclaims supported by\nretrieved context?\n→ LLM-as-judge: \"Is\nthis claim supported\nby the context?\"\n• Relevance: does the\nanswer address the\nquestion?\n• Completeness: did\nthe answer cover all\naspects?\n• Hallucination rate:\n% of claims with no\nsource support","originalText":"Dimension 2:\nGeneration Quality\n• Faithfulness: are\nclaims supported by\nretrieved context?\n→ LLM-as-judge: \"Is\nthis claim supported\nby the context?\"\n• Relevance: does the\nanswer address the\nquestion?\n• Completeness: did\nthe answer cover all\naspects?\n• Hallucination rate:\n% of claims with no\nsource support","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":1257,"width":288,"height":379.2,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":1275,"width":248,"height":343.2,"fontSize":22,"fontFamily":1,"text":"Dimension 3:\nEnd-to-End\n• User satisfaction:\nthumbs up/down ratio\n• Answer accepted\nrate: did user take\naction based on\nanswer?\n• Follow-up rate: high\nfollow-ups =\nincomplete first\nanswer","originalText":"Dimension 3:\nEnd-to-End\n• User satisfaction:\nthumbs up/down ratio\n• Answer accepted\nrate: did user take\naction based on\nanswer?\n• Follow-up rate: high\nfollow-ups =\nincomplete first\nanswer","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":1680.2,"width":288,"height":493.6,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":1698.2,"width":248,"height":457.6,"fontSize":22,"fontFamily":1,"text":"Automated\nevaluation pipeline\n(nightly):\n1. Run test suite of\n500+ (query,\nexpected_answer,\nsources)\n2. Compare with\nprevious day's\nresults\n3. Alert if any\ndimension regresses >\n5%\n4. Track per-change:\n\"after adding BM25,\nrecall improved 12%\"","originalText":"Automated\nevaluation pipeline\n(nightly):\n1. Run test suite of\n500+ (query,\nexpected_answer,\nsources)\n2. Compare with\nprevious day's\nresults\n3. Alert if any\ndimension regresses >\n5%\n4. Track per-change:\n\"after adding BM25,\nrecall improved 12%\"","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":2217.8,"width":288,"height":522.2,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":2235.8,"width":248,"height":486.2,"fontSize":22,"fontFamily":1,"text":"Continuous\nimprovement loop:\nUser feedback →\nidentify failure\npatterns →\n→ Chunking too\ncoarse? → reduce\nchunk size\n→ Wrong documents\nretrieved? → improve\nembedding model\n→ Right docs but\nwrong answer? →\nimprove prompt\n→ Missing documents\nentirely? → check\ningestion pipeline","originalText":"Continuous\nimprovement loop:\nUser feedback →\nidentify failure\npatterns →\n→ Chunking too\ncoarse? → reduce\nchunk size\n→ Wrong documents\nretrieved? → improve\nembedding model\n→ Right docs but\nwrong answer? →\nimprove prompt\n→ Missing documents\nentirely? → check\ningestion pipeline","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}