{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":16,"y":16,"width":288,"height":68,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":36,"y":35.7,"width":248,"height":28.6,"fontSize":22,"fontFamily":1,"text":"Re-Crawl Scheduling","originalText":"Re-Crawl Scheduling","textAlign":"left","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":16,"y":128,"width":288,"height":179,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":36,"y":146,"width":248,"height":143,"fontSize":22,"fontFamily":1,"text":"Not all pages change\nat the same rate.\nRe-crawl frequency\nshould adapt to\nchange frequency.","originalText":"Not all pages change\nat the same rate.\nRe-crawl frequency\nshould adapt to\nchange frequency.","textAlign":"left","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":16,"y":351,"width":288,"height":93.2,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":36,"y":369,"width":248,"height":57.2,"fontSize":22,"fontFamily":1,"text":"Strategy: Adaptive\nRe-Crawl","originalText":"Strategy: Adaptive\nRe-Crawl","textAlign":"left","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":16,"y":488.2,"width":288,"height":322,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":36,"y":506.2,"width":248,"height":286,"fontSize":22,"fontFamily":1,"text":"Track per-URL:\nlast_crawled:\ntimestamp\ncontent_hash:\nSHA-256\nchange_count: int\n(how many times\ncontent changed\nacross crawls)\ncrawl_count: int","originalText":"Track per-URL:\nlast_crawled:\ntimestamp\ncontent_hash:\nSHA-256\nchange_count: int\n(how many times\ncontent changed\nacross crawls)\ncrawl_count: int","textAlign":"left","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":16,"y":854.2,"width":288,"height":121.8,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":36,"y":872.2,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"change_rate =\nchange_count /\ncrawl_count","originalText":"change_rate =\nchange_count /\ncrawl_count","textAlign":"left","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":16,"y":1020,"width":288,"height":379.2,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":36,"y":1038,"width":248,"height":343.2,"fontSize":22,"fontFamily":1,"text":"If change_rate >\n0.8: re-crawl every 1\nday\nIf change_rate >\n0.3: re-crawl every 7\ndays\nIf change_rate >\n0.1: re-crawl every 14\ndays\nIf change_rate <=\n0.1: re-crawl every\n30 days","originalText":"If change_rate >\n0.8: re-crawl every 1\nday\nIf change_rate >\n0.3: re-crawl every 7\ndays\nIf change_rate >\n0.1: re-crawl every 14\ndays\nIf change_rate <=\n0.1: re-crawl every\n30 days","textAlign":"left","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":16,"y":1443.2,"width":288,"height":121.8,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":36,"y":1461.2,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Conditional HTTP\nRequests (save\nbandwidth):","originalText":"Conditional HTTP\nRequests (save\nbandwidth):","textAlign":"left","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":16,"y":1609,"width":288,"height":179,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":36,"y":1627,"width":248,"height":143,"fontSize":22,"fontFamily":1,"text":"GET /page HTTP/1.1\nIf-Modified-Since:\nWed, 01 Jan 2026 ...\nIf-None-Match:\n\"etag-abc123\"","originalText":"GET /page HTTP/1.1\nIf-Modified-Since:\nWed, 01 Jan 2026 ...\nIf-None-Match:\n\"etag-abc123\"","textAlign":"left","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":16,"y":1832,"width":288,"height":207.6,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":36,"y":1850,"width":248,"height":171.6,"fontSize":22,"fontFamily":1,"text":"Server responds:\n304 Not Modified →\nskip download, save\n200 OK → page\nchanged, process\nnormally","originalText":"Server responds:\n304 Not Modified →\nskip download, save\n200 OK → page\nchanged, process\nnormally","textAlign":"left","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":16,"y":2083.6,"width":288,"height":121.8,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":36,"y":2101.6,"width":248,"height":85.8,"fontSize":22,"fontFamily":1,"text":"Saves ~40% of\nbandwidth for\nre-crawls","originalText":"Saves ~40% of\nbandwidth for\nre-crawls","textAlign":"left","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}