[{"data":1,"prerenderedAt":719},["ShallowReactive",2],{"\u002Farticles\u002F3":3},{"id":4,"title":5,"body":6,"category":707,"cover":708,"date":709,"description":710,"extension":711,"meta":712,"navigation":713,"path":714,"seo":715,"sitemap":716,"stem":717,"__hash__":718},"articles\u002Farticles\u002F3万条会员被分配了两次-同步延迟背锅案的平反.md","3 万条会员被分配了两次：一次\"同步延迟\"背锅案的平反",{"type":7,"value":8,"toc":692},"minimark",[9,14,18,47,50,57,60,64,67,79,86,89,95,99,102,120,126,132,138,141,145,148,248,253,260,263,277,283,286,293,297,326,329,340,355,361,365,368,371,389,396,400,403,410,414,417,420,424,431,434,440,474,481,495,501,511,514,518,521,608,611,632,636,639,644,655,662,665,668,676,679,685,688],[10,11,13],"h2",{"id":12},"一事情是这样的","一、事情是这样的",[15,16,17],"p",{},"前阵子一个朋友找我聊问题。他们公司做 CRM 系统，技术栈大概是：",[19,20,21,29,35,41],"ul",{},[22,23,24,28],"li",{},[25,26,27],"strong",{},"MySQL 8","：业务主库，订单、会员、分配关系等源数据都在这",[22,30,31,34],{},[25,32,33],{},"ClickHouse","：会员数据的复杂组合条件查询引擎",[22,36,37,40],{},[25,38,39],{},"Redis 7","：缓存",[22,42,43,46],{},[25,44,45],{},"阿里 DTS","：MySQL 增量同步到 ClickHouse，频率大概 2000 条\u002F秒",[15,48,49],{},"为什么上 ClickHouse？因为 CRM 的会员圈选查询条件非常复杂——会员属性、订单行为、话务记录、访问明细多表组合，MySQL 扛不住这种任意组合的筛选，所以引入了 CK 做查询加速。",[15,51,52,53,56],{},"架构看起来很标准：",[25,54,55],{},"写在 MySQL，读在 ClickHouse，中间一条同步链路","。",[15,58,59],{},"然后出事了。",[10,61,63],{"id":62},"二事故现象任务包里的会员重复了","二、事故现象：任务包里的会员重复了",[15,65,66],{},"他们的业务流程大概是：",[68,69,70,73,76],"ol",{},[22,71,72],{},"操作员用组合条件在 CK 里圈选会员，一捞就是十几万",[22,74,75],{},"从里面挑 3~5 万，分配到不同的任务包（公海 → 租户 → 员工）",[22,77,78],{},"分配动作全部在 MySQL 上做：更新分配关系表、记录日志、写任务包",[15,80,81,82,85],{},"最近发现：",[25,83,84],{},"部分任务包里出现了重复的会员","。同一个会员，被分进了不止一个任务包。",[15,87,88],{},"朋友的初步排查结论是：DTS 同步有延迟，MySQL 里已经分配掉的会员，CK 里还是\"公海\"状态，下一轮圈选又把它捞出来了。",[15,90,91,92],{},"这个判断对不对？",[25,93,94],{},"对了一半。",[10,96,98],{"id":97},"三根因分析同步延迟只是帮凶不是主犯","三、根因分析：同步延迟只是帮凶，不是主犯",[15,100,101],{},"我跟朋友说，如果只盯着同步延迟修，大概率修不干净。重复数据的产生，需要三个条件同时成立：",[15,103,104,107,108,112,113,115,116,119],{},[25,105,106],{},"条件 1：读写分离带来的时间差。"," 分配发生在 MySQL，圈选发生在 CK。DTS 2000 条\u002F秒，而且一次分配涉及多张表变更，延迟是必然的——秒级甚至分钟级。CK 里用了 ",[109,110,111],"code",{},"FINAL"," 查询也没用，",[109,114,111],{}," 解决的是 ReplacingMergeTree 合并前的多行问题，",[25,117,118],{},"解决不了\"数据还没同步过来\"的问题","。这两件事经常被混为一谈。",[15,121,122,125],{},[25,123,124],{},"条件 2：分配动作没有防重保护。"," 流程是\"CK 查出列表 → 直接批量 UPDATE\"。没有任何一行代码去校验\"这个会员现在到底是不是还能分\"。只要 CK 给了，就照单全收。",[15,127,128,131],{},[25,129,130],{},"条件 3：并发操作没有互斥。"," 操作员经常是\"先捞十几万，再慢慢分\"。两个人圈选条件有重叠时，就算同步零延迟，抢同一批数据也会出重复。",[15,133,134,135],{},"所以结论很明确：",[25,136,137],{},"就算把 DTS 优化到毫秒级，只要写入不做条件校验、并发不做控制，重复照样发生。",[15,139,140],{},"修复的重心，不应该放在同步链路上。",[10,142,144],{"id":143},"四方案讨论先别急着动手把候选方案摆上桌","四、方案讨论：先别急着动手，把候选方案摆上桌",[15,146,147],{},"我们列了五个候选方案，逐个过：",[149,150,151,170],"table",{},[152,153,154],"thead",{},[155,156,157,161,164,167],"tr",{},[158,159,160],"th",{},"方案",[158,162,163],{},"思路",[158,165,166],{},"能否根治重复",[158,168,169],{},"成本",[171,172,173,191,207,221,235],"tbody",{},[155,174,175,179,185,188],{},[176,177,178],"td",{},"① CK 查询加时间偏移过滤",[176,180,181,184],{},[109,182,183],{},"update_time \u003C NOW() - 5min","，制造\"安全窗口\"",[176,186,187],{},"❌ 逻辑不成立",[176,189,190],{},"低",[155,192,193,196,202,205],{},[176,194,195],{},"② MySQL 条件写（CAS）",[176,197,198,201],{},[109,199,200],{},"UPDATE ... WHERE status='未分配'","，校验 affected_rows",[176,203,204],{},"✅ 根治",[176,206,190],{},[155,208,209,212,215,218],{},[176,210,211],{},"③ 预分配 + 黑名单",[176,213,214],{},"MySQL 锁定状态 + Redis 过滤\"锁定中\"ID",[176,216,217],{},"✅ 辅助防重",[176,219,220],{},"中",[155,222,223,226,229,232],{},[176,224,225],{},"④ 优化同步链路",[176,227,228],{},"换 Canal\u002FFlink CDC，压低延迟",[176,230,231],{},"❌ 治标",[176,233,234],{},"高",[155,236,237,240,243,246],{},[176,238,239],{},"⑤ 异步任务队列",[176,241,242],{},"分配变成任务单，Worker 查 MySQL 主库执行",[176,244,245],{},"✅ 长期方向",[176,247,234],{},[249,250,252],"h3",{"id":251},"方案一个漂亮的错误","方案①：一个漂亮的错误",[15,254,255,256,259],{},"这是朋友从某个 AI 助手那里拿到的\"最快见效\"方案：查 CK 时加上 ",[109,257,258],{},"update_time \u003C NOW() - INTERVAL 5 MINUTE","，把最近 5 分钟变更过的数据排除掉。",[15,261,262],{},"乍一听很合理，仔细一想是错的：",[264,265,266],"blockquote",{},[15,267,268,269,272,273,276],{},"会员 X 十分钟前在 MySQL 被分配了，但这次变更因为 DTS 积压",[25,270,271],{},"还没同步到 CK","。CK 里 X 的 ",[109,274,275],{},"update_time"," 还是上次变更的旧时间——可能是半年前。旧时间通过了\"5 分钟前\"的过滤，X 照样被查出来。",[15,278,279,282],{},[25,280,281],{},"这个过滤器过滤掉的，是已经正确同步的数据；漏掉的，恰恰是有问题的数据。"," 方向完全反了。",[15,284,285],{},"唯一成立的变体是用\"同步入库时间\"过滤（CK 表里加一个 ETL 打点的 sync_time 字段），但那要改同步链路，就不是\"纯查询修改\"了。",[15,287,288,289,292],{},"这件事给我一个提醒：",[25,290,291],{},"AI 给的方案，每一条都要在脑子里跑一遍真实数据流，才能判断它是不是真的成立。"," 看起来逻辑通顺，不代表经得起推敲。",[249,294,296],{"id":295},"方案真正的主角mysql-条件写","方案②：真正的主角，MySQL 条件写",[298,299,304],"pre",{"className":300,"code":301,"language":302,"meta":303,"style":303},"language-sql shiki shiki-themes github-dark","UPDATE 会员分配关系表\nSET 租户ID = ?, 状态 = '已分配', 更新时间 = NOW()\nWHERE member_id IN (...) AND 状态 = '未分配';\n","sql","",[109,305,306,314,320],{"__ignoreMap":303},[307,308,311],"span",{"class":309,"line":310},"line",1,[307,312,313],{},"UPDATE 会员分配关系表\n",[307,315,317],{"class":309,"line":316},2,[307,318,319],{},"SET 租户ID = ?, 状态 = '已分配', 更新时间 = NOW()\n",[307,321,323],{"class":309,"line":322},3,[307,324,325],{},"WHERE member_id IN (...) AND 状态 = '未分配';\n",[15,327,328],{},"不管 CK 查出来的列表有多\"脏\"，写入 MySQL 时带上\"仍是未分配\"这个条件，重复就物理上不可能发生——第二个分配请求的 affected_rows 会是 0。",[15,330,331,332,335,336,339],{},"这里有个",[25,333,334],{},"他们业务特有的坑","：他们的分配关系表是\"有记录才算归属，没记录就是公海\"。也就是说公海会员在表里根本没有行，分配动作实际是 ",[25,337,338],{},"INSERT 而不是 UPDATE","。那么防重就得靠：",[19,341,342,348],{},[22,343,344,347],{},[109,345,346],{},"member_id"," 唯一约束（数据库兜底，最硬）",[22,349,350,351,354],{},"或 ",[109,352,353],{},"INSERT ... SELECT ... WHERE NOT EXISTS","（代码兜底）",[15,356,357,358],{},"这个细节，无论是我还是那个 AI 助手，第一版方案里都漏掉了。",[25,359,360],{},"脱离具体表结构谈方案，都是纸上谈兵。",[249,362,364],{"id":363},"方案黑名单是体验优化不是正确性保障","方案③：黑名单是体验优化，不是正确性保障",[15,366,367],{},"思路是：分配前先在 MySQL 里把状态改成\"锁定中\"，同时把 ID 写进 Redis；别人查 CK 之后，用 Redis 黑名单把\"锁定中\"的过滤掉，就不会看到别人正在抢的数据。",[15,369,370],{},"两个细节必须抠：",[68,372,373,383],{},[22,374,375,378,379,382],{},[25,376,377],{},"黑名单过滤要放在应用层做","，CK 查完回内存里比对，不要拼成 ",[109,380,381],{},"id NOT IN (几万个ID)"," 塞进 CK 的 SQL——SQL 文本爆炸，解析和执行都慢。",[22,384,385,388],{},[25,386,387],{},"Redis 锁要按 ID 独立设置、各自带 TTL","，别用一个大 Set 统一设过期时间，否则要么锁被意外续命，要么一批锁同时蒸发。",[15,390,391,392,395],{},"另外要想明白主次关系：",[25,393,394],{},"黑名单丢了、Redis 挂了，都不会产生重复数据","——因为方案②的条件写在兜底。黑名单只是减少\"看得见却分不了\"的空转体验。想清楚这一点，就不会在这个方案上过度设计。",[249,397,399],{"id":398},"方案同步优化先量化再决定","方案④：同步优化，先量化再决定",[15,401,402],{},"DTS 到底延迟多少？峰值多少？朋友其实没有准确数字，只是\"感觉慢\"。",[15,404,405,406,409],{},"我的建议是：",[25,407,408],{},"先上延迟监控打点，拿到真实数字再决定要不要换链路","。如果延迟只有 3 秒，那方案②③组合就够了，换 Canal\u002FFlink CDC 的投入完全不值。监控是免费的，换链路是昂贵的。",[249,411,413],{"id":412},"方案异步任务化留给未来","方案⑤：异步任务化，留给未来",[15,415,416],{},"多人高频大批量分配，终态应该是\"提交任务单 → 后台 Worker 查 MySQL 主库执行 → 任务中心看结果\"。Worker 直查主库，天然没有延迟问题。",[15,418,419],{},"但它要改前端交互、引入任务中心，属于架构级改造，不适合在救火期做。",[10,421,423],{"id":422},"五加状态字段的风险别只盯着分配这一条链路","五、加状态字段的风险：别只盯着分配这一条链路",[15,425,426,427,430],{},"方案③要引入\"锁定中\"这个新状态，朋友当时最担心的就是这个。担心得对——",[25,428,429],{},"会员分配关系表是他们系统里变化最频繁的表","，公海转租户、内部归属变更、回收、延迟、黑名单，全都要读写这张表。",[15,432,433],{},"加一个状态值，影响面必须一条条过：",[15,435,436,439],{},[25,437,438],{},"1. 多次更新带来的中间态暴露。"," \"未分配 → 锁定中 → 已分配\"从一次写变成两次写。第二次写之前进程崩了怎么办？必须配套超时释放的定时任务。这里有两个设计要点：",[19,441,442,460],{},[22,443,444,447,448,451,452,455,456,459],{},[25,445,446],{},"时间字段建议存\"过期时刻\"而不是\"加锁时刻\"","。与其存 ",[109,449,450],{},"lock_time"," 再算 ",[109,453,454],{},"lock_time \u003C NOW() - 10min","，不如加锁时直接写入 ",[109,457,458],{},"lock_expire_time = NOW() + TTL","。好处有两个：不同批量大小可以给不同 TTL；长任务可以通过续期（更新 expire_time）保活，避免\"任务还在跑，锁被端了\"。",[22,461,462,465,466,469,470,473],{},[25,463,464],{},"防误伤靠的不是时间字段，而是条件写的原子性","。超时释放和分配完成是两条争抢同一行的 UPDATE，MySQL 行锁会串行化它们，先执行者生效：分配先完成，释放 SQL 的 ",[109,467,468],{},"WHERE status='锁定中'"," 匹配 0 行，误伤不了；释放先执行，分配 SQL 匹配 0 行，失败被暴露。所以两条 SQL 都必须带 ",[109,471,472],{},"status='锁定中'"," 条件，谁输谁 affected_rows=0，谁也别想悄悄改赢。",[15,475,476,477,480],{},"还有一个更隐蔽的边界要防：",[25,478,479],{},"静默丢失","。任务跑得太慢被定时任务回收，数据被别人合法分走，姗姗来迟的\"完成\"SQL 匹配 0 行——这时如果代码不看 affected_rows，前端还提示\"分配成功\"，数据就对不上了。所以完成侧必须校验 affected_rows，少于预期就反查哪些会员被回收了，明确告知操作员，绝不能退化成按 ID 强更新。",[15,482,483,486,487,490,491,494],{},[25,484,485],{},"2. 存量查询条件的排查成本。"," 所有查这张表的地方，凡是写了 ",[109,488,489],{},"status = '未分配'"," 的，都要问一句：",[25,492,493],{},"\"锁定中\"的数据该不该被查出来？"," 回收流程查\"可回收\"列表、黑名单流程查\"在册\"会员、报表统计\"公海数量\"——每一处都是一个判断点。漏改一处，就是新的线上问题。",[15,496,497,500],{},[25,498,499],{},"3. 对 DTS 同步的额外压力。"," 锁定操作本身也是一次写，会产生 binlog，DTS 要多同步一批数据。高频分配场景下，这等于给本来就有积压的同步链路雪上加霜。好在锁定状态只是过程态，CK 端查询最终依赖 MySQL 兜底，这个压力可以接受，但要心里有数。",[15,502,503,506,507,510],{},[25,504,505],{},"4. 状态回滚的路径要想全。"," 分配成功：锁定中 → 已分配。分配失败：锁定中 → 未分配，且要清锁信息。操作员中途关浏览器：靠定时任务超时释放。",[25,508,509],{},"每一条失败路径都要有对应的状态出口","，否则数据会被永久冻结在\"锁定中\"，业务方会拿着刀来找你。",[15,512,513],{},"这就是现实的工程判断：加状态字段不是加一个枚举值的事，是把一颗石子扔进一张蛛网，每一根丝都会动。",[10,515,517],{"id":516},"六改造成本与时间评估","六、改造成本与时间评估",[15,519,520],{},"和朋友最后敲定的落地计划，按投入产出排序：",[149,522,523,539],{},[152,524,525],{},[155,526,527,530,533,536],{},[158,528,529],{},"阶段",[158,531,532],{},"内容",[158,534,535],{},"预估工时",[158,537,538],{},"上线时间",[171,540,541,555,569,581,594],{},[155,542,543,546,549,552],{},[176,544,545],{},"🔴 紧急止血",[176,547,548],{},"分配写入改为条件写 + affected_rows 校验 + 确认 INSERT 防重（唯一约束）",[176,550,551],{},"1~2 人日",[176,553,554],{},"本周内",[155,556,557,560,563,566],{},[176,558,559],{},"🟡 体验加固",[176,561,562],{},"Redis 锁定标记（应用层过滤）+ 超时释放定时任务",[176,564,565],{},"3~4 人日",[176,567,568],{},"下周",[155,570,571,574,577,579],{},[176,572,573],{},"🟡 安全网",[176,575,576],{},"对账任务：扫描跨任务包重复会员，出报表\u002F修复存量脏数据（可选）",[176,578,551],{},[176,580,568],{},[155,582,583,586,589,592],{},[176,584,585],{},"🟢 观测",[176,587,588],{},"DTS 同步延迟监控打点 + 告警",[176,590,591],{},"1 人日",[176,593,568],{},[155,595,596,599,602,605],{},[176,597,598],{},"⚪ 排期",[176,600,601],{},"分配任务异步化（任务单 + Worker）",[176,603,604],{},"2~3 周",[176,606,607],{},"下个迭代评估",[15,609,610],{},"几个关键判断：",[19,612,613,619,626],{},[22,614,615,618],{},[25,616,617],{},"止血方案只改分配服务层，不动表结构、不动同步链路、不动 CK","，所以可以当天评审、隔天上线。救火期最重要的就是改动面小。",[22,620,621,622,625],{},"对账任务看起来是\"锦上添花\"，其实是",[25,623,624],{},"必做项","——存量已经产生的重复数据不会自己消失，而且它还能兜住未来任何漏网的新场景。",[22,627,628,629],{},"异步化先不做。等业务量真的涨到并发冲突成为日常，再启动也不迟。",[25,630,631],{},"过早的架构升级，本身就是一种风险。",[10,633,635],{"id":634},"七写在最后","七、写在最后",[15,637,638],{},"这个问题前前后后聊了两天，最大的感受是：",[15,640,641],{},[25,642,643],{},"这类一致性问题的答案，从来不是某个精妙的技术方案，而是一连串清醒的取舍。",[19,645,646,649,652],{},[22,647,648],{},"同步延迟要不要治？先量化，别凭感觉。",[22,650,651],{},"新状态字段要不要加？先把所有读写这张表的流程列一遍。",[22,653,654],{},"架构要不要升级？先把止血做完，让子弹飞一会儿。",[15,656,657,658,661],{},"写 MySQL 读 CK 的架构没有错，错的是默认\"查出来的就是最新的\"。",[25,659,660],{},"任何跨存储的读写分离，都要先回答一个问题：当两边不一致时，谁说了算？"," 在他们的场景里，答案很清晰——分配状态，永远以 MySQL 为准；CK 只是一个筛选引擎。",[15,663,664],{},"把这个原则写进团队规范，比修十个 Bug 都值钱。",[15,666,667],{},"最后用一句话收尾，也是我跟朋友说的原话：",[264,669,670],{},[15,671,672,675],{},[25,673,674],{},"没有永远完美的方案，只有最适合当前的取舍。"," 工程的美学不在\"最优解\"里，而在\"清楚地知道自己放弃了什么\"里。",[677,678],"hr",{},[15,680,681],{},[682,683,684],"em",{},"如果你也遇到过读写分离带来的一致性问题，欢迎留言聊聊你们最后选了哪条路。有更好建议，也希望留言给我，感谢",[15,686,687],{},"标签：#Java #ClickHouse #DTS #读写分离 #脏数据",[689,690,691],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}",{"title":303,"searchDepth":316,"depth":316,"links":693},[694,695,696,697,704,705,706],{"id":12,"depth":316,"text":13},{"id":62,"depth":316,"text":63},{"id":97,"depth":316,"text":98},{"id":143,"depth":316,"text":144,"children":698},[699,700,701,702,703],{"id":251,"depth":322,"text":252},{"id":295,"depth":322,"text":296},{"id":363,"depth":322,"text":364},{"id":398,"depth":322,"text":399},{"id":412,"depth":322,"text":413},{"id":422,"depth":316,"text":423},{"id":516,"depth":316,"text":517},{"id":634,"depth":316,"text":635},"article","\u002Fimg\u002Fmysql-to-clickhouse.png","2026-05-20","写 MySQL、查 ClickHouse，DTS同步延迟只是帮凶。一次生产问题的原因分析和思考。没有永远完美的方案，只有最适合当前的取舍。","md",{},true,"\u002Farticles\u002F3",{"title":5,"description":710},{"loc":714},"articles\u002F3万条会员被分配了两次-同步延迟背锅案的平反","JNvTiST5nn1fsZBXKogIquhDQCC4SlYGv36xDm2yRy0",1793263560455]