{"type":"excalidraw","version":2,"source":"https://AwesomeSystemDesign.dev","elements":[{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-1"},{"type":"arrow","id":"shape-26"},{"type":"arrow","id":"shape-27"},{"type":"arrow","id":"shape-28"}],"updated":0,"link":null,"locked":false,"id":"shape-0","type":"rectangle","seed":41,"x":377.50793999999996,"y":32.00000000000001,"width":221.77516,"height":82.80000000000001,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-1"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-1","type":"text","seed":114,"x":397.50793999999996,"y":50.000000000000014,"width":181.77516,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Load Balancer\n(L7, multi-AZ, TLS)","originalText":"Load Balancer\n(L7, multi-AZ, TLS)","textAlign":"center","verticalAlign":"middle","containerId":"shape-0","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":1}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-28","type":"arrow","seed":2085,"x":377.50793999999996,"y":99.03763032711524,"width":245.50793999999996,"height":97.76236967288477,"points":[[0,0],[-245.50793999999996,56.76236967288477],[-245.50793999999996,97.76236967288477]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-0","focus":0,"gap":1},"endBinding":{"elementId":"shape-2","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-3"},{"type":"arrow","id":"shape-28"},{"type":"arrow","id":"shape-30"},{"type":"arrow","id":"shape-33"},{"type":"arrow","id":"shape-36"}],"updated":0,"link":null,"locked":false,"id":"shape-2","type":"rectangle","seed":187,"x":36,"y":196.8,"width":192,"height":199.8,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-2"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-3","type":"text","seed":260,"x":56,"y":214.8,"width":152,"height":163.8,"fontSize":18,"fontFamily":1,"text":"API Gateway\n(AZ-c)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","originalText":"API Gateway\n(AZ-c)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","textAlign":"center","verticalAlign":"middle","containerId":"shape-2","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":2}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-27","type":"arrow","seed":2012,"x":547.6641167233009,"y":114.80000000000001,"width":58.695953276699015,"height":82,"points":[[0,0],[58.695953276699015,41],[58.695953276699015,82]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-0","focus":0,"gap":1},"endBinding":{"elementId":"shape-4","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-5"},{"type":"arrow","id":"shape-27"},{"type":"arrow","id":"shape-29"},{"type":"arrow","id":"shape-32"},{"type":"arrow","id":"shape-35"}],"updated":0,"link":null,"locked":false,"id":"shape-4","type":"rectangle","seed":333,"x":510.36006999999995,"y":196.8,"width":192,"height":199.8,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-3"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-5","type":"text","seed":406,"x":530.36007,"y":214.8,"width":152,"height":163.8,"fontSize":18,"fontFamily":1,"text":"API Gateway\n(AZ-b)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","originalText":"API Gateway\n(AZ-b)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","textAlign":"center","verticalAlign":"middle","containerId":"shape-4","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":3}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-26","type":"arrow","seed":1939,"x":599.2831,"y":88.82589364595381,"width":481.4370399999999,"height":107.9741063540462,"points":[[0,0],[481.4370399999999,66.9741063540462],[481.4370399999999,107.9741063540462]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-0","focus":0,"gap":1},"endBinding":{"elementId":"shape-6","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-7"},{"type":"arrow","id":"shape-26"},{"type":"arrow","id":"shape-31"},{"type":"arrow","id":"shape-34"},{"type":"arrow","id":"shape-37"}],"updated":0,"link":null,"locked":false,"id":"shape-6","type":"rectangle","seed":479,"x":984.7201399999999,"y":196.8,"width":192,"height":199.8,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-4"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-7","type":"text","seed":552,"x":1004.7201399999999,"y":214.8,"width":152,"height":163.8,"fontSize":18,"fontFamily":1,"text":"API Gateway\n(AZ-a)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","originalText":"API Gateway\n(AZ-a)\n\n• Rate limiting\n• Auth\n• Idempotency\ndedup check","textAlign":"center","verticalAlign":"middle","containerId":"shape-6","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":4}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-37","type":"arrow","seed":2742,"x":1000.6016162242724,"y":396.59999999999997,"width":747.6370662242724,"height":227.10405763468128,"points":[[0,0],[-32.88147622427255,41.00000000000006],[-747.6370662242724,227.10405763468128]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-6","focus":0,"gap":1},"endBinding":{"elementId":"shape-8","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-36","type":"arrow","seed":2669,"x":117.81973030518098,"y":396.59999999999997,"width":36.146300971103756,"height":210.7000000000001,"points":[[0,0],[-5.8197303051809826,41.00000000000006],[30.326570665922773,210.7000000000001]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-8","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-35","type":"arrow","seed":2596,"x":510.36006999999995,"y":336.32502116899616,"width":332.20808451918515,"height":270.9749788310039,"points":[[0,0],[-245.36006999999995,101.27497883100386],[-332.20808451918515,270.9749788310039]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-4","focus":0,"gap":1},"endBinding":{"elementId":"shape-8","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-9"},{"type":"arrow","id":"shape-35"},{"type":"arrow","id":"shape-36"},{"type":"arrow","id":"shape-37"}],"updated":0,"link":null,"locked":false,"id":"shape-8","type":"rectangle","seed":625,"x":60.96455000000003,"y":607.3000000000001,"width":192,"height":82.80000000000001,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-5"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-9","type":"text","seed":698,"x":80.96455000000003,"y":625.3000000000001,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Cron Evaluator\n× 3 nodes","originalText":"Cron Evaluator\n× 3 nodes","textAlign":"center","verticalAlign":"middle","containerId":"shape-8","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":5}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-34","type":"arrow","seed":2523,"x":1014.7818859190915,"y":396.59999999999997,"width":345.4225759190915,"height":175.53081116012498,"points":[[0,0],[-27.06174591909155,41.00000000000006],[-345.4225759190915,175.53081116012498]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-6","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-33","type":"arrow","seed":2450,"x":197.93825408090842,"y":396.59999999999997,"width":121.92968941980214,"height":117.10000000000008,"points":[[0,0],[27.061745919091578,41.00000000000006],[121.92968941980214,117.10000000000008]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-32","type":"arrow","seed":2377,"x":599.154585748758,"y":396.59999999999997,"width":41.90319222436199,"height":117.10000000000008,"points":[[0,0],[-2.9572057487580423,41.00000000000006],[-41.90319222436199,117.10000000000008]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-4","focus":0,"gap":1},"endBinding":{"elementId":"shape-10","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-11"},{"type":"arrow","id":"shape-32"},{"type":"arrow","id":"shape-33"},{"type":"arrow","id":"shape-34"},{"type":"arrow","id":"shape-38"}],"updated":0,"link":null,"locked":false,"id":"shape-10","type":"rectangle","seed":771,"x":306.96455000000003,"y":513.7,"width":362.39475999999996,"height":270,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-6"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-11","type":"text","seed":844,"x":326.96455000000003,"y":531.7,"width":322.39475999999996,"height":234,"fontSize":18,"fontFamily":1,"text":"Queue Layer\n(Redis Cluster)\n\n3 master + 3 replica\n\n• ready:high ready:medium ready:low\n\n• delayed (sorted set)\n\n• processing (in-flight tracking)","originalText":"Queue Layer\n(Redis Cluster)\n\n3 master + 3 replica\n\n• ready:high ready:medium ready:low\n\n• delayed (sorted set)\n\n• processing (in-flight tracking)","textAlign":"center","verticalAlign":"middle","containerId":"shape-10","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":6}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-31","type":"arrow","seed":2304,"x":1094.900409694819,"y":396.59999999999997,"width":144.71006999999986,"height":158.02738374040763,"points":[[0,0],[5.819730305180883,41.00000000000006],[-138.89033969481898,158.02738374040763]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-6","focus":0,"gap":1},"endBinding":{"elementId":"shape-12","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-30","type":"arrow","seed":2231,"x":212.11852377572745,"y":396.59999999999997,"width":511.2407862242726,"height":210.8070457152681,"points":[[0,0],[32.88147622427255,41.00000000000006],[511.2407862242726,210.8070457152681]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-2","focus":0,"gap":1},"endBinding":{"elementId":"shape-12","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-29","type":"arrow","seed":2158,"x":702.36007,"y":336.32502116899616,"width":245.36006999999995,"height":142.27497883100386,"points":[[0,0],[245.36006999999995,101.27497883100386],[224.377344040739,142.27497883100386]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-4","focus":0,"gap":1},"endBinding":{"elementId":"shape-12","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-13"},{"type":"arrow","id":"shape-29"},{"type":"arrow","id":"shape-30"},{"type":"arrow","id":"shape-31"},{"type":"arrow","id":"shape-40"}],"updated":0,"link":null,"locked":false,"id":"shape-12","type":"rectangle","seed":917,"x":723.35931,"y":478.6,"width":232.65076,"height":340.2,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-7"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-13","type":"text","seed":990,"x":743.35931,"y":496.6,"width":192.65076,"height":304.2,"fontSize":18,"fontFamily":1,"text":"Task Store (DB)\n\nSharded PostgreSQL\n(20 shards)\n\n• Task records\n• Cron definitions\n• Workflow state\n• Results\n• DLQ\n\nPrimary + standby\nper shard","originalText":"Task Store (DB)\n\nSharded PostgreSQL\n(20 shards)\n\n• Task records\n• Cron definitions\n• Workflow state\n• Results\n• DLQ\n\nPrimary + standby\nper shard","textAlign":"center","verticalAlign":"middle","containerId":"shape-12","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":7}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-15"},{"type":"arrow","id":"shape-39"}],"updated":0,"link":null,"locked":false,"id":"shape-14","type":"rectangle","seed":1063,"x":1010.01007,"y":607.3000000000001,"width":211.49104,"height":82.80000000000001,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-8"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-15","type":"text","seed":1136,"x":1030.01007,"y":625.3000000000001,"width":171.49104,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Timeout Watchdog\n× 2 nodes","originalText":"Timeout Watchdog\n× 2 nodes","textAlign":"center","verticalAlign":"middle","containerId":"shape-14","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":8}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-40","type":"arrow","seed":2961,"x":839.68469,"y":818.8000000000001,"width":9.696701948336226,"height":81.99999999999989,"points":[[0,0],[0,40.999999999999886],[9.696701948336226,81.99999999999989]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-12","focus":0,"gap":1},"endBinding":{"elementId":"shape-16","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-39","type":"arrow","seed":2888,"x":1115.75559,"y":690.1,"width":15.081990915061397,"height":210.69999999999993,"points":[[0,0],[0,169.69999999999993],[-15.081990915061397,210.69999999999993]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-14","focus":0,"gap":1},"endBinding":{"elementId":"shape-16","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#7c899d","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.4,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-38","type":"arrow","seed":2815,"x":488.16193,"y":783.7,"width":264.5582099999999,"height":339.0703652296843,"points":[[0,0],[0,76.09999999999991],[264.5582099999999,339.0703652296843]],"lastCommittedPoint":null,"startBinding":{"elementId":"shape-10","focus":0,"gap":1},"endBinding":{"elementId":"shape-16","focus":0,"gap":1},"startArrowhead":null,"endArrowhead":"arrow","elbowed":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-17"},{"type":"arrow","id":"shape-38"},{"type":"arrow","id":"shape-39"},{"type":"arrow","id":"shape-40"}],"updated":0,"link":null,"locked":false,"id":"shape-16","type":"rectangle","seed":1209,"x":752.7201399999999,"y":900.8,"width":390,"height":831.6,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-9"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-17","type":"text","seed":1282,"x":772.7201399999999,"y":918.8,"width":350,"height":795.6,"fontSize":18,"fontFamily":1,"text":"WORKER FLEET\n\nAuto-scaling group (Kubernetes HPA\non queue depth)\nMin: 200 workers | Max: 5,000\nworkers | Target: queue_depth <\n10K\n\nWorker types:\n• Generic: processes any task type\n• Specialized: only processes specific\ntask types (for isolation)\ne.g., payment workers separate from\nemail workers\n\n• Worker 1\n[email] [report]\nThread Pool: 10\n\n• Worker 2\n[payment] [email]\nThread Pool: 10\n\n• Worker 3\n[email] [webhook]\nThread Pool: 10\n\n• Worker 4\n[report] [payment]\nThread Pool: 10\n\n• Worker N\n[any]\nThread Pool: 10","originalText":"WORKER FLEET\n\nAuto-scaling group (Kubernetes HPA\non queue depth)\nMin: 200 workers | Max: 5,000\nworkers | Target: queue_depth <\n10K\n\nWorker types:\n• Generic: processes any task type\n• Specialized: only processes specific\ntask types (for isolation)\ne.g., payment workers separate from\nemail workers\n\n• Worker 1\n[email] [report]\nThread Pool: 10\n\n• Worker 2\n[payment] [email]\nThread Pool: 10\n\n• Worker 3\n[email] [webhook]\nThread Pool: 10\n\n• Worker 4\n[report] [payment]\nThread Pool: 10\n\n• Worker N\n[any]\nThread Pool: 10","textAlign":"center","verticalAlign":"middle","containerId":"shape-16","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":9}},{"angle":0,"strokeColor":"#628572","backgroundColor":"#edf5ee","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-19"}],"updated":0,"link":null,"locked":false,"id":"shape-18","type":"rectangle","seed":1355,"x":36,"y":1846.3999999999999,"width":192,"height":82.80000000000001,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-10"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-19","type":"text","seed":1428,"x":56,"y":1864.3999999999999,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"Delayed Poller\n× 2 nodes","originalText":"Delayed Poller\n× 2 nodes","textAlign":"center","verticalAlign":"middle","containerId":"shape-18","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":10}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-21"}],"updated":0,"link":null,"locked":false,"id":"shape-20","type":"rectangle","seed":1501,"x":340,"y":1846.3999999999999,"width":192,"height":82.80000000000001,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-11"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-21","type":"text","seed":1574,"x":360,"y":1864.3999999999999,"width":152,"height":46.800000000000004,"fontSize":18,"fontFamily":1,"text":"DAG Engine\n× 2 nodes","originalText":"DAG Engine\n× 2 nodes","textAlign":"center","verticalAlign":"middle","containerId":"shape-20","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":11}},{"angle":0,"strokeColor":"#9180ac","backgroundColor":"#f4effb","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-23"}],"updated":0,"link":null,"locked":false,"id":"shape-22","type":"rectangle","seed":1647,"x":538,"y":2043.2,"width":390,"height":363.6,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-12"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-23","type":"text","seed":1720,"x":558,"y":2061.2,"width":350,"height":327.6,"fontSize":18,"fontFamily":1,"text":"trace_id: tr-abc123\nspan: API receive (2ms)\nspan: DB persist (5ms)\nspan: Redis enqueue (1ms)\n[gap: 45ms in queue]\nspan: Worker pickup (1ms)\nspan: Task execute (1185ms)\nspan: Fetch user from DB (20ms)\nspan: Render email template (15ms)\nspan: Send via SMTP (1150ms)\nspan: Result persist (3ms)\n\nTotal: 1242ms (of which 1185ms was\nactual task work)","originalText":"trace_id: tr-abc123\nspan: API receive (2ms)\nspan: DB persist (5ms)\nspan: Redis enqueue (1ms)\n[gap: 45ms in queue]\nspan: Worker pickup (1ms)\nspan: Task execute (1185ms)\nspan: Fetch user from DB (20ms)\nspan: Render email template (15ms)\nspan: Send via SMTP (1150ms)\nspan: Result persist (3ms)\n\nTotal: 1242ms (of which 1185ms was\nactual task work)","textAlign":"center","verticalAlign":"middle","containerId":"shape-22","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":12}},{"angle":0,"strokeColor":"#6577aa","backgroundColor":"#eef1fc","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-25"}],"updated":0,"link":null,"locked":false,"id":"shape-24","type":"rectangle","seed":1793,"x":36,"y":2043.1999999999998,"width":390,"height":901.8000000000001,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-13"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-25","type":"text","seed":1866,"x":56,"y":2061.2,"width":350,"height":865.8000000000001,"fontSize":18,"fontFamily":1,"text":"Metrics (Prometheus / Datadog):\n\nAlerts:\nRED — DLQ depth > 0 (tasks\npermanently failing)\nRED — Queue depth growing > 10%\nper minute (sustained)\nRED — Task execution error rate >\n5%\nRED — Worker fleet < 50% healthy\nWARN — Queue depth > 100K\nWARN — Average scheduling latency\n> 1 second\nWARN — Retry rate > 10% for any\ntask type\nWARN — Cron fire drift > 5 seconds\n\n• Task submission rate (by type, by\ntenant)\n• Task execution rate (by type, by\nworker)\n• Queue depth (by queue, by priority)\n• Task latency: submission-to-start,\nexecution duration\n• Retry rate (by type — indicates\nunhealthy downstream)\n• DLQ depth (by queue — requires\nhuman attention)\n• Worker utilization (busy threads /\ntotal threads)\n• Cron fire accuracy\n(actual_fire_time -\nscheduled_fire_time)\n• DAG completion time (by workflow\ntype)\n• Lease expiration rate (indicates\nworker crashes)","originalText":"Metrics (Prometheus / Datadog):\n\nAlerts:\nRED — DLQ depth > 0 (tasks\npermanently failing)\nRED — Queue depth growing > 10%\nper minute (sustained)\nRED — Task execution error rate >\n5%\nRED — Worker fleet < 50% healthy\nWARN — Queue depth > 100K\nWARN — Average scheduling latency\n> 1 second\nWARN — Retry rate > 10% for any\ntask type\nWARN — Cron fire drift > 5 seconds\n\n• Task submission rate (by type, by\ntenant)\n• Task execution rate (by type, by\nworker)\n• Queue depth (by queue, by priority)\n• Task latency: submission-to-start,\nexecution duration\n• Retry rate (by type — indicates\nunhealthy downstream)\n• DLQ depth (by queue — requires\nhuman attention)\n• Worker utilization (busy threads /\ntotal threads)\n• Cron fire accuracy\n(actual_fire_time -\nscheduled_fire_time)\n• DAG completion time (by workflow\ntype)\n• Lease expiration rate (indicates\nworker crashes)","textAlign":"center","verticalAlign":"middle","containerId":"shape-24","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":13}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-14"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-42"}],"updated":0,"link":null,"locked":false,"id":"shape-41","type":"rectangle","seed":3034,"x":36,"y":2985,"width":420,"height":68,"customData":{"revealStep":14}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-14"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-42","type":"text","seed":3107,"x":56,"y":3007.95,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Scheduler Nodes","originalText":"Scheduler Nodes","textAlign":"left","verticalAlign":"middle","containerId":"shape-41","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":14}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-15"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-44"}],"updated":0,"link":null,"locked":false,"id":"shape-43","type":"rectangle","seed":3180,"x":36,"y":3077,"width":420,"height":68,"customData":{"revealStep":15}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-15"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-44","type":"text","seed":3253,"x":56,"y":3099.95,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Monitoring & Observability:","originalText":"Monitoring & Observability:","textAlign":"left","verticalAlign":"middle","containerId":"shape-43","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":15}},{"angle":0,"strokeColor":"#a68a53","backgroundColor":"#fff7e6","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-16"],"frameId":null,"roundness":null,"boundElements":[{"type":"text","id":"shape-46"}],"updated":0,"link":null,"locked":false,"id":"shape-45","type":"rectangle","seed":3326,"x":36,"y":3169,"width":420,"height":68,"customData":{"revealStep":16}},{"angle":0,"strokeColor":"#394557","backgroundColor":"transparent","fillStyle":"solid","strokeWidth":1.5,"strokeStyle":"solid","roughness":0.7,"opacity":100,"version":1,"versionNonce":1,"isDeleted":false,"groupIds":["step-16"],"frameId":null,"roundness":null,"boundElements":null,"updated":0,"link":null,"locked":false,"id":"shape-46","type":"text","seed":3399,"x":56,"y":3191.95,"width":380,"height":22.1,"fontSize":17,"fontFamily":1,"text":"Distributed Tracing (per task):","originalText":"Distributed Tracing (per task):","textAlign":"left","verticalAlign":"middle","containerId":"shape-45","lineHeight":1.3,"autoResize":false,"customData":{"revealStep":16}}],"appState":{"viewBackgroundColor":"#fbfbf8","gridSize":null},"files":{}}