From cae3f2177ce9506b7f3b87e565db0bec699cabe7 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Mon, 22 Jun 2026 13:17:21 +0800 Subject: [PATCH 01/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0rebalance=E5=88=86?= =?UTF-8?q?=E6=9E=90=E6=96=87=E7=AB=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- ...25\345\261\202\345\216\237\347\220\206.md" | 66 -- ...73\350\276\221\345\210\206\346\236\220.md" | 794 ++++++++++++++++++ 3 files changed, 795 insertions(+), 67 deletions(-) delete mode 100644 "note/Netty/IO\345\272\225\345\261\202\345\216\237\347\220\206.md" create mode 100644 "note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" diff --git a/README.md b/README.md index 1e01d4c..eeffec2 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,6 @@ Java流行框架源码分析,学习以及总结。项目持续更新中,不 - Netty底层源码解析-FastThreadLocal原理分析 - Netty底层源码解析-内存分配原理分析 - Netty底层源码解析-RocketMQ底层使用到的Netty - - [Netty底层的优化总结]() - [实战+原理效果更佳!强烈推荐闪电侠大佬实战课:《Netty 入门与实战:仿写微信 IM 即时通讯系统》](https://juejin.cn/book/6844733738119593991) Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下,有需要的读者可前往查看。 @@ -156,6 +155,7 @@ Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下, - kafka源码分析 - kafka版本:4.2 - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) + - [kafka broker核心源码分析——生产者篇]() - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) 持续更新中... diff --git "a/note/Netty/IO\345\272\225\345\261\202\345\216\237\347\220\206.md" "b/note/Netty/IO\345\272\225\345\261\202\345\216\237\347\220\206.md" deleted file mode 100644 index b66a558..0000000 --- "a/note/Netty/IO\345\272\225\345\261\202\345\216\237\347\220\206.md" +++ /dev/null @@ -1,66 +0,0 @@ -## 从linux kernel内核出发,IO底层原理 - -### 1. BIO - -``` -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; - -/** - * @author lhy - * - * 在windows服务器下,可以使用telnet来合serversocket建立连接 - */ -public class BIO { - public static void main(String[] args) throws Exception { - ServerSocket serverSocket = new ServerSocket(666); - System.out.println("Server started..."); - while (true) { - System.out.println("socket accepting..."); - Socket socket = serverSocket.accept(); - new Thread(new Runnable() { - @Override - public void run() { - try { - byte[] bytes = new byte[1024]; - InputStream inputStream = socket.getInputStream(); - while (true) { - System.out.println("reading..."); - int read = inputStream.read(bytes); - if (read != -1) { - System.out.println(new String(bytes, 0, read)); - } else { - break; - } - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - try { - socket.close(); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - }).start(); - } - } -} -``` - -#### 1.1 从kernel内核的角度来分析BIO的运行机制 - -### 2. IO多路复用 - -#### 2.1 select - -#### 2.2 poll - -#### 2.3 epoll - -### 3. 零拷贝 - - - diff --git "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" new file mode 100644 index 0000000..20fbee5 --- /dev/null +++ "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" @@ -0,0 +1,794 @@ +# Kafka Rebalance 核心流程 +`KafkaConsumer.poll()` 是消费者触发 rebalance 的主要入口。 + +位置在 [KafkaConsumer.java (line 916)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java:916): + +```plain +public ConsumerRecords poll(final Duration timeout) { + return delegate.poll(timeout); +} +``` + +在 classic consumer 实现里,会进入 [ClassicKafkaConsumer.poll() (line 641)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ClassicKafkaConsumer.java:641): + +```plain +KafkaConsumer.poll + -> ClassicKafkaConsumer.poll + -> updateAssignmentMetadataIfNeeded + -> ConsumerCoordinator.poll + -> updateFetchPositions + -> pollForFetches +``` + +也就是说,consumer 每次 `poll()` 拉数据之前,都会先确认自己是否已经完成消费组协调、分区分配和 fetch offset 准备。 + +`ClassicKafkaConsumer.poll()` 最核心做了三件事: + ++ **推进消费组协调** +调用 `updateAssignmentMetadataIfNeeded(...)`,确保自动分配分区的 consumer 已经加入 group,并且 rebalance 已经完成。 ++ **准备 fetch 位置** +如果分区还没有有效 position,会根据 committed offset、auto offset reset 等规则确定从哪里开始拉。 ++ **拉取并返回消息** +从 fetch buffer 取数据,或者发送新的 FetchRequest,并把拿到的 records 返回给用户。 + +对应 [ClassicKafkaConsumer.java (line 665)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ClassicKafkaConsumer.java:665): + +```plain +updateAssignmentMetadataIfNeeded(timer, false) +pollForFetches(timer) +sendFetches() +``` + +再精简一句:**rebalance 不是后台自动完整完成的,classic consumer 的 group rebalance 主要是在用户线程调用 **`poll()`** 时被推进的。** + +--- + +## ConsumerCoordinator.poll() +`ConsumerCoordinator.poll()` 是 consumer 端消费组协调的入口。 + +位置在 [ConsumerCoordinator.java (line 530)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:530): + +```plain +public boolean poll(Timer timer, boolean waitForJoinGroup) +``` + +这个方法最核心做了五件事: + ++ **更新订阅元数据** +调用 `maybeUpdateSubscriptionMetadata()`,如果 metadata version 变了,就更新本地 `metadataSnapshot`。 ++ **执行已完成的 offset commit 回调** +异步提交的 callback 不一定在请求完成时立即由用户线程执行,这里统一触发。 ++ **维护心跳和 coordinator 连接** +刷新 heartbeat 的 last poll 时间;如果不知道 coordinator,就先找 coordinator。 ++ **必要时触发重新入组** +如果 `rejoinNeededOrPending()` 为 true,就调用 `ensureActiveGroup(...)` 推进 JoinGroup/SyncGroup。 ++ **处理自动提交** +如果开启 auto commit,并且提交周期到了,就异步提交当前消费位点。 + +核心链路是: + +```plain +ConsumerCoordinator.poll + -> maybeUpdateSubscriptionMetadata + -> invokeCompletedOffsetCommitCallbacks + -> pollHeartbeat + -> coordinatorUnknownAndUnreadySync + -> rejoinNeededOrPending + -> ensureActiveGroup + -> maybeAutoCommitOffsetsAsync +``` + +`waitForJoinGroup` 决定这次是否等待 rebalance 完成: + ++ `true`:使用用户传进来的 timer,可以阻塞等待入组完成。 ++ `false`:使用 0ms timer,只推进一次流程,不强等完成。 + +这也是为什么 `poll()` 可能先返回空数据:不是没有数据,而是本轮还在推进 rebalance 或准备 assignment。 + +### invokeCompletedOffsetCommitCallbacks()为什么要在poll()里执行? +invokeCompletedOffsetCommitCallbacks的核心作用是把已经完成的异步 offset commit 结果取出来,并调用用户传入的 OffsetCommitCallback.onComplete(...)。offset commit提交的是:某个consumer group对某个 topic-partition已经消费到哪里,它写入的是内部topic __consumer_offsets。invokeCompletedOffsetCommitCallbacks放在poll()里有几个原因: + +1. 保证 callback 在用户线程执行 + +Kafka 不希望在内部网络处理路径里直接跑用户 callback,因为用户 callback 可能很慢、可能抛异常、可能调用 consumer API。 + +2. 避免阻塞内部协议处理 + +如果在 response handler 里直接执行用户 callback,callback 慢会拖慢 coordinator response 处理,影响 rebalance、heartbeat、commit 等内部状态推进。 + +3. 及时暴露 fenced 异常 + +如果异步 commit 遇到 FencedInstanceIdException,代码会设置 asyncCommitFenced,下一次 invokeCompletedOffsetCommitCallbacks() 会抛出 fenced 异常。这也是为什么它要在 poll() 这种高频入口前置检查。fenced 可以理解成 身份被抢占后的隔离机制。配了 group.instance.id 后,consumer 变成静态成员。同一个 group.id + group.instance.id 同一时间只能有一个实例有效,fenced就是排查是否有重复的 group.instance.id 实例在运行。 + + + +### rejoinNeededOrPending() +这个方法命名就很有意思“是否需要重新加入消费组,或者重新加入消费组的流程仍在进行中”。首先来看请求第一次进来。 + +核心链路: + +```plain +ConsumerCoordinator.rejoinNeededOrPending + -> 判断是否是自动分配分区 + -> 判断metadataSnapshot + -> 判断subscriptions +``` + + + +如果用户是手动分配分区,就不需要rebalance了。 + +```java +if (!subscriptions.hasAutoAssignedPartitions()) + return false; +``` + + + +上次 assignment 使用的 metadata 和当前 metadata 不匹配,或者说集群 topic/partition 等元数据变化,需要重新分配。 + +```java +if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) { + ... + return true; +} +``` + + + +上次 JoinGroup 提交的 subscription 和当前 subscription 不一致,或者说用户订阅或正则匹配结果变化,需要重新入组。 + +```java +if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) { + ... + return true; +} +``` + + + +这里两处判断为啥都是上次的JoinGroup?因为这是在构造这一次JoinGroup之前,所以得那上次的JoinGroup的元数据和订阅信息和当前准备构造JoinGroup的数据进行对比。 + + + +如果上面两个判断都为false,则会走最后一个判断rejoinNeededOrPending + +```java + protected synchronized boolean rejoinNeededOrPending() { + return rejoinNeeded || joinFuture != null; + } +``` + +第一次发起JoinGroup,joinFuture=null。而rejoinNeeded为true是什么场景呢?这主要是通过AbstractCoordinator.requestRejoin()来触发 + +```java +public synchronized void requestRejoin(final String shortReason, + final String fullReason) { + log.info("Request joining group due to: {}", fullReason); + this.rejoinReason = shortReason; + this.rejoinNeeded = true; +} +``` + +而requestRejoin()的调用场景有:broker 返回 REBALANCE_IN_PROGRESS / ILLEGAL_GENERATION / UNKNOWN_MEMBER_ID。 + + + +## Rebalance触发条件? +是否需要加入或重新加入消费组,主要由 `rejoinNeededOrPending()` 判断。 + +位置在 [ConsumerCoordinator.java (line 953)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:953): + +```plain +public boolean rejoinNeededOrPending() +``` + +常见触发条件: + ++ **第一次加入消费组** +consumer 还没有 generation/member id,必须 JoinGroup。 ++ **订阅发生变化** +比如用户调用 `subscribe()` 改了 topic 列表。 ++ **metadata 发生变化** +topic 新增分区、删除分区、正则订阅匹配结果变化等。 ++ **成员变化** +有 consumer 加入、退出、session timeout、max.poll.interval.ms 超时。 ++ **cooperative rebalance 需要第二轮确认** +如果本轮 assignment 要 revoke 部分分区,consumer 会先 revoke,然后 request rejoin,下一轮再完成新分配。 + +--- + +## ensureActiveGroup() +`ensureActiveGroup()` 是“确保消费组已经可用”的核心方法。 + +位置在 [AbstractCoordinator.java (line 422)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:422): + +```plain +boolean ensureActiveGroup(final Timer timer) { + if (!ensureCoordinatorReady(timer)) { + return false; + } + startHeartbeatThreadIfNeeded(); + return joinGroupIfNeeded(timer); +} +``` + +这个方法最核心做了三件事: + ++ **找到 coordinator** +如果还不知道 group coordinator 是哪个 broker,就通过 FindCoordinator 找到它。 ++ **启动 heartbeat 线程** +heartbeat 线程负责维持 session,避免 broker 认为该 member 死掉。 ++ **必要时加入 group** +调用 `joinGroupIfNeeded(...)` 推进 JoinGroup 和 SyncGroup。 + +简化链路: + +```plain +ensureActiveGroup + -> ensureCoordinatorReady + -> startHeartbeatThreadIfNeeded + -> joinGroupIfNeeded +``` + +注意:heartbeat 线程负责维持成员活性,但 rebalance 的 JoinGroup/SyncGroup 主流程仍然由调用 `poll()` 的用户线程推进。 + +--- + +## joinGroupIfNeeded() +`joinGroupIfNeeded()` 是 consumer 端 rebalance 状态机的核心。 + +位置在 [AbstractCoordinator.java (line 476)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:476): + +```plain +boolean joinGroupIfNeeded(final Timer timer) +``` + +核心流程: + +```plain +joinGroupIfNeeded + -> while rejoinNeededOrPending + -> ensureCoordinatorReady + -> onJoinPrepare + -> initiateJoinGroup + -> sendJoinGroupRequest + -> JoinGroupResponseHandler + -> sendSyncGroupRequest + -> SyncGroupResponseHandler + -> client.poll(future, timer) + -> onJoinComplete +``` + +这个方法最核心做了四件事: + ++ **入组前准备** +调用 `onJoinPrepare(...)`,处理 auto commit、revoke/lost callback、清理旧 assignment。 ++ **发送 JoinGroup** +把自己的订阅信息、支持的 assignor、当前拥有的 partitions 发给 broker coordinator。 ++ **发送 SyncGroup** +如果自己是 leader,先计算整个 group 的 assignment,再发 SyncGroup;如果是 follower,发送空 assignment 等 leader 的结果。 ++ **应用新 assignment** +SyncGroup 成功后,调用 `onJoinComplete(...)`,更新本地分配并触发 assigned callback。 + +关键点:`joinGroupIfNeeded()` 会循环执行,直到 rebalance 成功、超时、被 wakeup,或者遇到不可恢复错误。 + +--- + +## JoinGroup 请求携带什么 +JoinGroup 请求里最关键的是 `metadata()` 生成的 subscription metadata。 + +位置在 [ConsumerCoordinator.java (line 303)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:303): + +```plain +protected JoinGroupRequestProtocolCollection metadata() +``` + +每个 assignor 都会生成一份协议元数据,里面包括: + ++ 当前订阅的 topics。 ++ assignor 自己的 user data。 ++ 当前 consumer 已经拥有的 partitions。 ++ 当前 generation id。 ++ rack id。 + +核心代码: + +```plain +Subscription subscription = new Subscription( + topics, + assignor.subscriptionUserData(joinedSubscription), + subscriptions.assignedPartitionsList(), + generation().generationId, + rackId +); +``` + +所以 JoinGroup 不是只告诉 broker “我订阅了哪些 topic”,还会告诉 broker “我上一次拥有哪些 partition”。这对 cooperative rebalance 非常关键。 + +--- + +## Broker 侧 JoinGroup / SyncGroup +broker 网络入口仍然是 `KafkaApis.handle()`。 + +位置在 [KafkaApis.scala (line 180)](/Users/apple/Documents/git_code_pr/kafka/core/src/main/scala/kafka/server/KafkaApis.scala:180): + +```plain +case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request, requestLocal).exceptionally(handleError) +case ApiKeys.SYNC_GROUP => handleSyncGroupRequest(request, requestLocal).exceptionally(handleError) +``` + +`handleJoinGroupRequest()` 会做 group 权限校验,然后交给 `groupCoordinator.joinGroup()`。 + +位置在 [KafkaApis.scala (line 1381)](/Users/apple/Documents/git_code_pr/kafka/core/src/main/scala/kafka/server/KafkaApis.scala:1381): + +```plain +groupCoordinator.joinGroup( + request.context, + joinGroupRequest.data, + requestLocal.bufferSupplier +) +``` + +`GroupCoordinatorService.joinGroup()` 最终调度 classic group 的 join 操作。 + +位置在 [GroupCoordinatorService.java (line 1053)](/Users/apple/Documents/git_code_pr/kafka/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java:1053): + +```plain +runtime.scheduleWriteOperation( + "classic-group-join", + topicPartitionFor(request.groupId()), + coordinator -> coordinator.classicGroupJoin(context, request, responseFuture) +) +``` + +`syncGroup()` 类似,会调度 classic group sync。 + +位置在 [GroupCoordinatorService.java (line 1125)](/Users/apple/Documents/git_code_pr/kafka/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java:1125): + +```plain +runtime.scheduleWriteOperation( + "classic-group-sync", + topicPartitionFor(request.groupId()), + coordinator -> coordinator.classicGroupSync(context, request, responseFuture) +) +``` + +broker coordinator 的职责可以概括成: + ++ 维护 group 成员、generation、leader、protocol。 ++ 收集所有成员的 JoinGroup metadata。 ++ 选出 leader。 ++ 等 leader 通过 SyncGroup 提交完整 assignment。 ++ 把每个 member 自己的 assignment 返回给它。 + +broker 不负责具体分区分配算法。classic consumer 的分配算法是在 leader consumer 本地执行的。 + +--- + +## Leader 如何执行分配 +当 JoinGroupResponse 返回后,如果当前 consumer 被 broker 选为 leader,会进入: + +```plain +JoinGroupResponseHandler.handle + -> onLeaderElected(joinResponse) + -> ConsumerCoordinator.onLeaderElected(...) +``` + +位置在 [AbstractCoordinator.java (line 724)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:724): + +```plain +if (joinResponse.isLeader()) { + onLeaderElected(joinResponse).chain(future); +} else { + onJoinFollower().chain(future); +} +``` + +leader 的核心逻辑在 [ConsumerCoordinator.onLeaderElected() (line 686)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:686): + +```plain +protected Map onLeaderElected(...) +``` + +这个方法最核心做了六件事: + ++ **反序列化所有成员的订阅信息** +从 JoinGroupResponse 的 `members` 里解析每个 member 的 `Subscription`。 ++ **收集 group 订阅的所有 topic** +用于更新 leader 的 metadata 监听范围。 ++ **收集 owned partitions** +保存每个 member 上报的旧 assignment。 ++ **调用 assignor 执行分配** +`assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions))`。 ++ **校验 cooperative assignment** +对自定义 cooperative assignor,调用 `validateCooperativeAssignment(...)` 防止直接抢占仍被别人持有的 partition。 ++ **序列化 assignment** +把每个 member 的 assignment 序列化,准备放进 leader SyncGroup。 + +核心代码在 [ConsumerCoordinator.java (line 724)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:724): + +```plain +Map assignments = + assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions)).groupAssignment(); +``` + +leader 计算完 assignment 后,会在 [AbstractCoordinator.java (line 836)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:836) 构造 leader SyncGroup: + +```plain +SyncGroupRequest.Builder requestBuilder = + new SyncGroupRequest.Builder( + new SyncGroupRequestData() + .setAssignments(groupAssignmentList) + ); +``` + +简化链路: + +```plain +JoinGroupResponse leader + -> deserialize all subscriptions + -> update group subscription metadata + -> assignor.assign(...) + -> validate cooperative assignment + -> serialize assignment per member + -> SyncGroupRequest(assignments) +``` + +--- + +## Follower 做什么 +follower 不计算 assignment。 + +位置在 [AbstractCoordinator.java (line 801)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:801): + +```plain +private RequestFuture onJoinFollower() +``` + +follower 只发送一个空 assignment 的 SyncGroup: + +```plain +setAssignments(Collections.emptyList()) +``` + +意思是:follower 告诉 broker “我已经进入 sync 阶段了,具体分配结果等 leader 提交”。 + +最终 broker 会在 SyncGroupResponse 里把属于这个 follower 的 assignment 返回给它。 + +--- + +## SyncGroupResponse 如何完成 Rebalance +SyncGroupResponse 成功后,会进入 `SyncGroupResponseHandler.handle(...)`。 + +位置在 [AbstractCoordinator.java (line 871)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:871): + +```plain +if (!hasGenerationReset(generation) && state == MemberState.COMPLETING_REBALANCE) { + state = MemberState.STABLE; + rejoinNeeded = false; + future.complete(ByteBuffer.wrap(syncResponse.data().assignment())); +} +``` + +这里最核心做了三件事: + ++ 校验 generation 没有被 heartbeat 线程重置。 ++ 将成员状态改成 `STABLE`。 ++ 把 broker 返回的 assignment 放进 future。 + +future 成功后,`joinGroupIfNeeded()` 会调用 `onJoinComplete(...)` 应用 assignment。 + +--- + +## onJoinPrepare() +`onJoinPrepare()` 是入组前的清理阶段。 + +位置在 [ConsumerCoordinator.java (line 785)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:785): + +```plain +protected boolean onJoinPrepare(Timer timer, int generation, String memberId) +``` + +这个方法最核心做了三件事: + ++ **入组前自动提交 offset** +如果开启 auto commit,会先尝试异步提交当前消费位点。 ++ **触发 revoke 或 lost callback** +如果 generation/member id 已经被重置,说明之前的分区已经 lost;否则按 EAGER/COOPERATIVE 协议触发 revoke。 ++ **清理旧 group subscription 状态** +重置 leader 标记和 group subscription,为下一轮 JoinGroup 做准备。 + +EAGER 协议下: + +```plain +revoke all currently assigned partitions +subscriptions.assignFromSubscribed(emptySet) +``` + +COOPERATIVE 协议下: + +```plain +only revoke partitions whose topics are no longer subscribed +keep still-owned partitions locally +``` + +这就是 EAGER 和 COOPERATIVE 的核心差异: + ++ EAGER:每次 rebalance 先全部撤销,再重新分配。 ++ COOPERATIVE:尽量保留仍然合法的分区,只撤销需要迁移的分区。 + +--- + +## onJoinComplete() +`onJoinComplete()` 是本地应用 assignment 的阶段。 + +位置在 [ConsumerCoordinator.java (line 380)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:380): + +```plain +protected void onJoinComplete(...) +``` + +这个方法最核心做了五件事: + ++ **反序列化 SyncGroupResponse 里的 assignment** +得到当前 member 最终分到的 partitions。 ++ **校验 assignment 是否匹配当前订阅** +如果 assignment 和当前 subscription 不一致,说明订阅可能变化了,直接 request rejoin。 ++ **计算 added/revoked partitions** +和本地当前 owned partitions 比较,得出新增和撤销集合。 ++ **COOPERATIVE 下处理第二阶段 revoke** +如果有 revoked partitions,先触发 revoke callback,然后 request rejoin,下一轮再完成迁移。 ++ **应用新 assignment 并触发 assigned callback** +调用 `subscriptions.assignFromSubscribed(...)`,然后触发 `onPartitionsAssigned(...)`。 + +核心代码在 [ConsumerCoordinator.java (line 421)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:421): + +```plain +SortedSet addedPartitions = new TreeSet<>(COMPARATOR); +addedPartitions.addAll(assignedPartitions); +addedPartitions.removeAll(ownedPartitions); +``` + +COOPERATIVE 下如果发现 revoked partitions: + +```plain +invokePartitionsRevoked(revokedPartitions) +requestRejoin("need to revoke partitions and re-join", ...) +``` + +最后应用 assignment: + +```plain +subscriptions.assignFromSubscribed(assignedPartitions) +invokePartitionsAssigned(addedPartitions) +``` + +也就是说,`onJoinComplete()` 才是用户真正看到分区变化的地方。 + +--- + +## validateCooperativeAssignment() +`validateCooperativeAssignment()` 只用于 COOPERATIVE rebalance。 + +位置在 [ConsumerCoordinator.java (line 757)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:757): + +```plain +private void validateCooperativeAssignment(...) +``` + +它的作用是校验自定义 cooperative assignor 是否违反规则。 + +COOPERATIVE 的基本规则是: + +```plain +如果 partition 当前还被 member A 拥有, +assignor 不能在同一轮 rebalance 里直接把它分给 member B。 +``` + +正确流程必须是: + +```plain +第一轮 rebalance: + A 先 revoke partition + 该 partition 暂时不分配给任何人 + +第二轮 rebalance: + partition 已经没人 owned + 再把它分给 B +``` + +`validateCooperativeAssignment()` 的判断方式: + +```plain +for each member: + added = newAssignment - oldOwnedPartitions + revoked = oldOwnedPartitions - newAssignment + +if totalAdded intersects totalRevoked: + illegal assignment +``` + +如果有交集,说明某个 partition 一边被旧 owner revoke,一边又被新 owner added。这就是“没有先释放,直接转移”,违反 COOPERATIVE 协议。 + +内置 `CooperativeStickyAssignor` 会跳过这个校验,位置在 [ConsumerCoordinator.java (line 726)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:726): + +```plain +if (protocol == RebalanceProtocol.COOPERATIVE && + !assignorName.equals(COOPERATIVE_STICKY_ASSIGNOR_NAME)) { + validateCooperativeAssignment(ownedPartitions, assignments); +} +``` + +原因是内置 cooperative sticky assignor 自己已经处理了 owned partition 的 generation 语义。 + +--- + +# EAGER 和 COOPERATIVE 的差异 +## EAGER +EAGER rebalance 的特点是简单直接。 + +```plain +onJoinPrepare: + revoke all partitions + clear local assignment + +leader assign: + assign all partitions again + +onJoinComplete: + assign new partitions + trigger onPartitionsAssigned +``` + +优点: + ++ 实现简单。 ++ assignment 一次完成。 + +缺点: + ++ 每次 rebalance 都会短暂停止所有分区消费。 ++ 即使某些 partition 仍然分给同一个 consumer,也会先 revoke 再 assign。 + +## COOPERATIVE +COOPERATIVE rebalance 的特点是渐进迁移。 + +```plain +onJoinPrepare: + keep still-owned partitions + revoke only obviously invalid partitions + +leader assign: + do not immediately reassign partitions still owned by others + +onJoinComplete: + revoke partitions that need migration + request another rejoin if needed + +next rebalance: + assign released partitions to new owners +``` + +优点: + ++ 减少分区停止消费的范围。 ++ 尽量保持稳定 assignment。 ++ 更适合大 group、大 partition 数量的场景。 + +缺点: + ++ 可能需要多轮 rebalance。 ++ assignor 必须遵守“先释放,再转移”的规则。 ++ 状态机更复杂。 + +--- + +# 一条完整 Rebalance 链路 +以自动订阅 consumer 触发 rebalance 为例: + +```plain +KafkaConsumer.poll + -> ClassicKafkaConsumer.poll + -> updateAssignmentMetadataIfNeeded + -> ConsumerCoordinator.poll + -> maybeUpdateSubscriptionMetadata + -> pollHeartbeat + -> rejoinNeededOrPending + -> ensureActiveGroup + -> ensureCoordinatorReady + -> startHeartbeatThreadIfNeeded + -> joinGroupIfNeeded + -> onJoinPrepare + -> maybe auto commit + -> revoke/lost old partitions + -> sendJoinGroupRequest + -> ConsumerCoordinator.metadata + -> JoinGroupRequest + -> broker KafkaApis.handleJoinGroupRequest + -> GroupCoordinatorService.joinGroup + -> select generation / leader / protocol + -> JoinGroupResponseHandler + -> leader: + ConsumerCoordinator.onLeaderElected + -> deserialize subscriptions + -> assignor.assign + -> validateCooperativeAssignment + -> serialize group assignment + leader SyncGroupRequest(assignments) + follower: + follower SyncGroupRequest(empty assignments) + -> broker KafkaApis.handleSyncGroupRequest + -> GroupCoordinatorService.syncGroup + -> return each member assignment + -> SyncGroupResponseHandler + -> state = STABLE + -> future.complete(assignment) + -> onJoinComplete + -> deserialize assignment + -> compute added/revoked partitions + -> cooperative may request another rejoin + -> subscriptions.assignFromSubscribed + -> onPartitionsAssigned + -> updateFetchPositions + -> pollForFetches +``` + +再压缩成一句: + +```plain +poll() + -> 确认是否要 rebalance + -> JoinGroup 上报订阅和 owned partitions + -> broker 选 leader + -> leader 本地执行 assignor.assign + -> SyncGroup 广播 assignment + -> 每个 member 应用自己的 assignment + -> 准备 offset 后开始 fetch +``` + +--- + +# 总结 +Kafka classic consumer rebalance 的核心可以理解成三层协作: + ++ **ConsumerCoordinator 负责客户端状态机** +判断是否需要 rejoin、入组前 revoke、发送 JoinGroup/SyncGroup、应用 assignment。 ++ **GroupCoordinator 负责服务端 group 管理** +维护成员、generation、leader、protocol,并把 leader 提交的 assignment 分发给各个 member。 ++ **ConsumerPartitionAssignor 负责分区分配算法** +真正决定 topic partitions 分给哪些 consumer。 + +最关键的几个方法: + +```plain +ClassicKafkaConsumer.poll +ConsumerCoordinator.poll +AbstractCoordinator.ensureActiveGroup +AbstractCoordinator.joinGroupIfNeeded +ConsumerCoordinator.onJoinPrepare +ConsumerCoordinator.metadata +ConsumerCoordinator.onLeaderElected +ConsumerCoordinator.validateCooperativeAssignment +AbstractCoordinator.SyncGroupResponseHandler +ConsumerCoordinator.onJoinComplete +``` + +EAGER 和 COOPERATIVE 的本质区别是: + ++ EAGER 是“先全部释放,再重新分配”。 ++ COOPERATIVE 是“能不动就不动,要迁移的分区先释放,下一轮再分给别人”。 + +所以看 rebalance 源码时,最重要的是抓住这条主线: + +```plain +是否需要重平衡 + -> 入组前怎么处理旧分区 + -> JoinGroup 带了哪些成员信息 + -> leader 如何计算 assignment + -> SyncGroup 如何下发 assignment + -> member 如何应用 assignment +``` + +只要这条链路清楚,`ConsumerCoordinator`、`AbstractCoordinator`、`GroupCoordinatorService` 里的大部分逻辑都能对应到具体阶段。 + From 7eacf9f957978581a2234e360342b6ceed2e58f3 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Mon, 22 Jun 2026 13:19:41 +0800 Subject: [PATCH 02/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eeffec2..fd54185 100644 --- a/README.md +++ b/README.md @@ -155,8 +155,8 @@ Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下, - kafka源码分析 - kafka版本:4.2 - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - - [kafka broker核心源码分析——生产者篇]() - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) + - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) 持续更新中... From 064e255b1b46e49e73354537e4a1626415b3de5f Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 23 Jun 2026 22:35:26 +0800 Subject: [PATCH 03/37] fix --- README.md | 13 ++++--------- ...\272\220\347\240\201\345\210\206\346\236\220.md" | 8 ++++---- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index fd54185..f34ae2b 100644 --- a/README.md +++ b/README.md @@ -27,11 +27,9 @@

-Java流行框架源码分析,学习以及总结。项目持续更新中,不建议直接Fork,欢迎star、watch。 +Java相关流行框架源码分析,学习以及总结,项目持续更新中。 -对于框架底层源码的学习,需要反复、认真思考,并做到温故而知新,这样才能将底层原理吸收得更加牢固。 - -框架包括: +框架或者源码包括: ✅ JDK源码 @@ -59,14 +57,11 @@ Java流行框架源码分析,学习以及总结。项目持续更新中,不 > 为什么要分析、学习源码? -学习一个框架的源码,不仅在实际使用时如果出现问题,可以快速定位出问题,找到问题原因并解决,同时还可以学习到框架的架构思想以与设计模式。当然,学习框架底层源码还可以提升我们自身的水平,在大厂面试时能够在众多面试者中脱颖而出。因此学习框架虽然枯燥乏味,但罗马并非一日建成的,所以平时之余就要多学习框架底层源码, -这样在用到的时候就能游刃有余。 + 学习框架源码不仅能帮助我们在实际问题出现时快速定位问题、理解根因并高效解决,还能深入掌握框架的整体架构设计思路与核心设计模式,从而提升自身的系统设计能力与架构思维。 +同时,通过学习优秀开源框架的底层实现,可以不断强化对复杂系统拆分、模块协作以及性能优化的理解,这对于个人技术能力的长期成长至关重要。因此,源码学习虽然过程相对枯燥,但这是提升架构设计能力的必经路径。只有持续积累,才能在实际系统设计与工程实践中做到真正的“游刃有余”。 # 目录 -- 项目导入 - 将整个JavaSourceCodeLearning导入IDEA中,然后选中项目pom.xml文件右键,open as maven project然后等待maven下载相应jar包即可。 - - JDK源码学习 - JDK版本:1.8.0_77 - [深入学习String源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) diff --git "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" index 037a16a..8556503 100644 --- "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -1,7 +1,7 @@ 当前分析版本是kafka最新版本 4.2(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) # kafka服务端核心逻辑 -目前最新版本kafka已经抛弃了Zookeeper,而是自己维护元数据日志,存放在内部 topic __cluster_metadata。 +目前最新版本kafka已经抛弃了Zookeeper,而是自己维护元数据日志,存放在内部 topic: __cluster_metadata。 如果使用默认的配置方式 config/server.properties启动: @@ -26,7 +26,7 @@ bin/kafka-server-start.sh 配置方式不同,启动链路不同: ```java -rocess.roles=broker +process.roles=broker ... KafkaRaftServer.startup() -> BrokerServer.startup() @@ -52,7 +52,7 @@ process.roles=broker,controller **** -**broker,controller部署方式适合本地开发。正式生产,适合独立 controller + 独立 broker,大规模生产适合独立 controller quorum集群 + 独立 broker 集群。** +**broker,controller部署方式适合本地开发。正式生产环境,适合独立部署controller + 独立部署broker,大规模生产适合独立 controller quorum集群 + 独立 broker 集群。** ## Kafka.scala Kafka.scala中main()是 Kafka broker 进程的 JVM 入口,它本身不做具体 broker 组件初始化,核心作用是把进程启动流程串起来,源码如下: @@ -150,7 +150,7 @@ socketServer = new SocketServer(config, val enableRequestProcessingFuture = socketServer.enableRequestProcessing(authorizerFutures) ``` -创建网络入口。此时准备 acceptor 和端口,真正处理请求会延后开启。enableRequestProcessing()开启 SocketServer 请求处理。 +SocketServer 用于创建网络入口。此时准备 acceptor 和端口,真正处理请求会延后开启。enableRequestProcessing()开启 SocketServer 请求处理。 From 4beb91db531fa3308f228511b07cf2b11f644d5f Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 24 Jun 2026 22:14:53 +0800 Subject: [PATCH 04/37] add --- README.md | 1 + ...37\344\272\247\350\200\205\347\257\207.md" | 783 ++++++++++++++++++ 2 files changed, 784 insertions(+) create mode 100644 "note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" diff --git a/README.md b/README.md index f34ae2b..4628f88 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,7 @@ Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下, - kafka源码分析 - kafka版本:4.2 - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) + - [kafka broker核心源码分析——生产者篇]() - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) diff --git "a/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" new file mode 100644 index 0000000..ea7feb9 --- /dev/null +++ "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" @@ -0,0 +1,783 @@ +# KafkaRequestHandler核心流程 +`KafkaRequestHandler.run()` 是在 **创建 **`**KafkaRequestHandlerPool**`** 时触发的**。 + +位置在 [KafkaRequestHandler.scala (line 261)]: + +```plain +private def createHandler(id: Int): Unit = { + runnables += new KafkaRequestHandler(...) + KafkaThread.daemon("data-plane-kafka-request-handler-" + id, runnables(id)).start() +} +``` + +`KafkaThread.daemon(...).start()` 会启动线程,然后进入: + +```plain +KafkaRequestHandler.run() +``` + +而 `KafkaRequestHandlerPool` 是在 [BrokerServer.startup() (line 515)]: + +```plain +dataPlaneRequestHandlerPool = sharedServer.requestHandlerPoolFactory.createPool(...) +``` + +所以启动链路是: + +```plain +BrokerServer.startup + -> create KafkaApis + -> create KafkaRequestHandlerPool + -> create KafkaRequestHandler + -> KafkaThread.start() + -> KafkaRequestHandler.run() +``` + +消费者 `FetchRequest` 从 broker 网络入口到 `KafkaRequestHandler` 的流转是: + +```plain +Producer 发送消息到 broker + +Acceptor.run + -> acceptNewConnections() + -> 接收 TCP 连接 + -> assignNewConnection(...) + -> Processor.accept(socketChannel) + +Processor.run + -> configureNewConnections() + 注册新连接到 selector + + -> poll() + 从 socket 读取网络数据 + + -> processCompletedReceives() + 解析 RequestHeader + 创建 RequestContext + 创建 Request + requestChannel.sendRequest(req) +``` + +关键代码在 [SocketServer.scala (line 1000)]: + +```plain +requestChannel.sendRequest(req) +selector.mute(connectionId) +``` + +`sendRequest` 会把请求放进 `RequestChannel` 的阻塞队列: + +```plain +requestQueue.put(request) +``` + +然后: + +```plain +KafkaRequestHandler.run + -> requestChannel.receiveRequest(300) + -> 取到 Processor 放进来的 Request + -> apis.handle(request, requestLocal) +``` + +对应 [KafkaRequestHandler.scala (line 117)]: + +```plain +val req = requestChannel.receiveRequest(300) +... +case request: Request => + apis.handle(request, requestLocal) +``` + +然后: + +```plain +KafkaApis.handle + -> 根据 request.header.apiKey 分发 + -> FetchRequest 命中 ApiKeys.FETCH + -> handleFetchRequest(request) +``` + +对应 [KafkaApis.scala (line 171)]: + +```plain +request.header.apiKey match { + case ApiKeys.PRODUCE => handleProduceRequest(request) +} +``` + +简化成一条链: + +```plain +Consumer ProduceRequest + -> SocketServer listener + -> Acceptor.acceptNewConnections + -> Processor.accept + -> Processor.run + -> processCompletedReceives + -> RequestChannel.sendRequest + -> KafkaRequestHandler.run + -> RequestChannel.receiveRequest + -> KafkaApis.handle + -> handleFetchRequest +``` + +Processor 负责把网络请求解析成 Request 并塞进 RequestChannel;KafkaRequestHandler 线程一直阻塞/轮询 RequestChannel,拿到请求后调用 KafkaApis.handle() 进入业务逻辑。 + +# KafkaApis.handle() +Kakfa producer在broker端,最核心的调用: + +```java +KafkaApis.handleProduceRequest() + -> ReplicaManager.handleProduceAppend() + -> ReplicaManager.appendRecords() + -> ReplicaManager.appendRecordsToLeader() + -> ReplicaManager.appendToLocalLog() + -> Partition.appendRecordsToLeader() + -> UnifiedLog.appendAsLeader() + -> UnifiedLog.append() + -> LocalLog.append() + -> LogSegment.append() + -> FileRecords.append() + -> MemoryRecords.writeFullyTo(FileChannel) +``` + + + +KafkaApis.handle() 是 broker API 分发入口。 + +核心源码: + +```java +override def handle(request: Request, requestLocal: RequestLocal): Unit = { + try { + if (!apiVersionManager.isApiEnabled(request.header.apiKey, request.header.apiVersion)) { + throw new IllegalStateException(...) + } + + request.header.apiKey match { + case ApiKeys.PRODUCE => handleProduceRequest(request, requestLocal) + case ApiKeys.FETCH => handleFetchRequest(request) + case ApiKeys.LIST_OFFSETS => handleListOffsetRequest(request) + case ApiKeys.METADATA => handleTopicMetadataRequest(request) + case ApiKeys.OFFSET_COMMIT => handleOffsetCommitRequest(request, requestLocal).exceptionally(handleError) + case ApiKeys.OFFSET_FETCH => handleOffsetFetchRequest(request).exceptionally(handleError) + case ApiKeys.FIND_COORDINATOR => handleFindCoordinatorRequest(request) + case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request, requestLocal).exceptionally(handleError) + case ApiKeys.HEARTBEAT => handleHeartbeatRequest(request).exceptionally(handleError) + ... + } + } +} +``` + + + +## KafkaApis.handleProduceRequest() +KafkaApis.handleProduceRequest() 最核心做了三件事: + ++ 校验请求能不能写,校验事务权限、Topic 写权限、Topic/Partition 是否存在、records 格式是否合法。 ++ 构造 Produce 响应返回客户端,合并成功/失败结果,处理限流、leader 信息、acks=0 特殊逻辑,然后返回或关闭连接。 ++ 把合法数据交给 ReplicaManager 写入日志,调用 replicaManager.handleProduceAppend(...),真正把消息追加到对应分区副本日志里,并按 acks 等语义等待结果。 + +总结,handleProduceRequest()是 Broker 处理生产者写消息请求的入口,负责校验、转交副本写入、返回写入结果。 + + + +```scala +if (RequestUtils.hasTransactionalRecords(produceRequest)) { + val isAuthorizedTransactional = produceRequest.transactionalId != null && + authHelper.authorize(request.context, WRITE, TRANSACTIONAL_ID, produceRequest.transactionalId) + if (!isAuthorizedTransactional) { + requestHelper.sendErrorResponseMaybeThrottle(request, Errors.TRANSACTIONAL_ID_AUTHORIZATION_FAILED.exception) + return + } +} +``` + +事务消息必须先校验 transactionalId 写权限,避免未授权写入进入副本追加流程。 + + + +```scala +@nowarn("cat=deprecation") +def sendResponseCallback(responseStatus: Map[TopicIdPartition, PartitionResponse]): Unit = { + ... + if (produceRequest.acks == 0) { + ... + if (errorInResponse) { + ... + requestChannel.closeConnection(request, new ProduceResponse(mergedResponseStatus.asJava).errorCounts) + } else { + requestHelper.sendNoOpResponseExemptThrottle(request) + } + } else { + requestChannel.sendResponse(request, new ProduceResponse(mergedResponseStatus.asJava, maxThrottleTimeMs, nodeEndpoints.values.toList.asJava)) + } +} +``` + + 构造并发送 Produce 响应,统一合并追加结果和前置校验失败结果。 + + + +```scala +if (authorizedRequestInfo.isEmpty) +sendResponseCallback(Map.empty) +else { + val internalTopicsAllowed = request.header.clientId == "__admin_client" + val transactionSupportedOperation = AddPartitionsToTxnManager.produceRequestVersionToTransactionSupportedOperation(request.header.apiVersion()) + // 交给 ReplicaManager 追加日志,并按 acks/min.insync.replicas 等语义完成响应。 + replicaManager.handleProduceAppend( + timeout = produceRequest.timeout.toLong, + requiredAcks = produceRequest.acks, + internalTopicsAllowed = internalTopicsAllowed, + transactionalId = produceRequest.transactionalId, + entriesPerPartition = authorizedRequestInfo, + responseCallback = sendResponseCallback, + recordValidationStatsCallback = processingStatsCallback, + requestLocal = requestLocal, + transactionSupportedOperation = transactionSupportedOperation) + + // 追加后清理请求中的 records 引用,避免延迟请求持有大对象影响 GC。 + produceRequest.clearPartitionRecords() +} +``` + +把合法数据交给 ReplicaManager 写入日志,调用 replicaManager.handleProduceAppend(...),真正把消息追加到对应分区副本日志里,并按 acks 等语义等待结果。 + +## ReplicaManager +### ReplicaManager.handleProduceAppend() +handleProduceAppend:处理 Produce请求入口, 这个方法最核心的三件事: + ++ 检查请求里有没有事务 batch。 ++ 如果是事务消息,先确认这些分区已经加入事务。 ++ 处理事务校验失败、重试、错误转换。 ++ 校验完成后再调用 `appendRecords()`。 + + + +```scala +val transactionalProducerInfo = mutable.HashSet[(Long, Short)]() +val topicPartitionBatchInfo = mutable.Map[TopicPartition, Int]() +val topicIds = entriesPerPartition.keys.map(tp => tp.topic() -> tp.topicId()).toMap +entriesPerPartition.foreachEntry { (topicIdPartition, records) => + // 提取事务 batch 的 producer 信息,后面用于校验这些分区是否已经加入事务。 + val transactionalBatches = records.batches.asScala.filter(batch => batch.hasProducerId && batch.isTransactional) + transactionalBatches.foreach(batch => transactionalProducerInfo.add(batch.producerId, batch.producerEpoch)) + if (transactionalBatches.nonEmpty) topicPartitionBatchInfo.put(topicIdPartition.topicPartition(), records.firstBatch.baseSequence) +} +if (transactionalProducerInfo.size > 1) { + throw new InvalidPidMappingException("Transactional records contained more than one producer ID") +} +``` + +检查请求里有没有事务 batch。 + + + +```scala +appendRecords( + timeout = timeout, + requiredAcks = requiredAcks, + internalTopicsAllowed = internalTopicsAllowed, + origin = AppendOrigin.CLIENT, + entriesPerPartition = entriesWithoutErrorsPerPartition, + responseCallback = newResponseCallback, + recordValidationStatsCallback = recordValidationStatsCallback, + requestLocal = newRequestLocal, + verificationGuards = verificationGuards +) +``` + +校验完成后再调用 appendRecords()。 + +### ReplicaManager.appendRecords() +这个方法核心作用就是:**把已经校验过的 Produce 数据追加到本 broker 作为 leader 的分区日志里,并根据 **`**acks**`** 判断是否需要等待副本复制后再返回。** + +拆开就是: + +1. **检查写入前置条件** +分区是否存在、当前 broker 是否 leader、`acks` 是否合法。 +2. **写入 leader 本地日志** +调用分区日志追加逻辑,把 records append 到 log。 +3. **处理响应时机** +如果不需要等待副本,立即回调;如果需要等 ISR 复制,则放入 delayed produce 等待完成或超时。 + + + +```scala + def appendRecords(timeout: Long, + requiredAcks: Short, + internalTopicsAllowed: Boolean, + origin: AppendOrigin, + entriesPerPartition: Map[TopicIdPartition, MemoryRecords], + responseCallback: util.Map[TopicIdPartition, PartitionResponse] => Unit, + recordValidationStatsCallback: Map[TopicIdPartition, RecordValidationStats] => Unit = _ => (), + requestLocal: RequestLocal = RequestLocal.noCaching, + verificationGuards: Map[TopicPartition, VerificationGuard] = Map.empty, + transactionVersion: Short = TransactionVersion.TV_UNKNOWN): Unit = { + // 核心总结:控制 Produce 追加的整体响应语义,先写 leader 本地日志,再按 acks 决定立即返回或等待复制。 + if (!isValidRequiredAcks(requiredAcks)) { + sendInvalidRequiredAcksResponse(entriesPerPartition, responseCallback) + return + } + + // 先完成 leader 本地追加,得到每个分区的初始写入结果。 + val localProduceResults = appendRecordsToLeader( + requiredAcks, + internalTopicsAllowed, + origin, + entriesPerPartition, + requestLocal, + defaultActionQueue, + verificationGuards, + transactionVersion + ) + + val produceStatus = buildProducePartitionStatus(localProduceResults) + + // 回传 record 校验/转换统计,用于 Produce 请求的处理指标。 + recordValidationStatsCallback(localProduceResults.map { case (k, v) => + k -> v.logAppendSummary().recordValidationStats() + }) + + // 根据 requiredAcks 判断是否需要进入 delayed produce 等待 ISR 副本追上。 + maybeAddDelayedProduce( + requiredAcks, + timeout, + entriesPerPartition, + localProduceResults, + produceStatus, + responseCallback + ) + } +``` + + + +### ReplicaManager.appendRecordsToLeader() +把一批 Produce records 写入当前 broker 上对应分区的 leader 本地日志,并返回每个分区的本地追加结果;然后调用 appendToLocalLog(),对 entriesPerPartition 逐分区处理,把 records 写到本地 leader replica 的 log。 + +```scala +val localProduceResultsWithTopicId = appendToLocalLog( + internalTopicsAllowed = internalTopicsAllowed, + origin, + entriesPerPartition, + requiredAcks, + requestLocal, + verificationGuards.toMap, + transactionVersion +) +``` + +entriesPerPartition是一个map类型的变量,表示这次请求要写入的哪些分区。遍历entriesPerPartition的过程是在appendToLocalLog()中完成的。 + + + +```scala +ddCompletePurgatoryAction(actionQueue, localProduceResultsWithTopicId) +``` + +这个动作会根据 leader HW 是否推进,唤醒可能已经满足条件的 delayed operations,例如: + + - delayed produce + +用于 Produce 请求,典型场景是 acks=all。leader 本地写成功后,还要等 ISR 副本复制到目标 offset。没满足前,请求挂起;满足后返回成功。如果超时,就按当前状态返回,比如 NOT_ENOUGH_REPLICAS_AFTER_APPEND。 + + - delayed fetch + +用于 Fetch 请求,包括消费者 fetch,也包括 follower 副本从 leader 拉数据。如果当前没有足够数据满足 fetch 条件,比如没有新消息、没达到 fetch.min.bytes、或者 high watermark 还没推进,请求会先挂起。等新数据写入或 HW 推进后,再检查是否可以返回。 + + - delayed delete records + +用于 DeleteRecords 请求。DeleteRecords 会推进分区的 logStartOffset,但它需要等相关副本状态或 high watermark 条件满足后才能完成。leader 写入或 HW 推进后,可能让挂起的 delete records 请求完成。 + + - delayed share fetch + +用于 Kafka Share Consumer / share group 相关的 fetch 请求。它和普通 fetch 类似,也是等待有可返回的数据或分区状态变化。区别是它服务的是 share group 的消费模型,而不是传统 consumer group 的普通 fetch。 + + + +在 appendRecordsToLeader() 之后唤醒它们,是因为一次 leader append 可能改变了这些条件: + + - LEO 增加 + - HW 推进 + - 有新数据可读 + - 某些等待 offset 的请求满足了 + +所以 broker 会让这些 delayed operation 重新检查自己是否可以完成。 + + + +小结:这个方法处在 Kafka 写入链路的中间层:它不是网络请求入口,也不是底层日志 append 的最终实现,而是 ReplicaManager 层的“批量写 leader 本地副本 + 收集结果 + 唤醒等待操作”的封装。 + +### ReplicaManager.appendToLocalLog() +遍历 Produce 请求中的各分区 records,找到本 broker 本地的 Partition 并委托给 Partition.appendRecordsToLeader() 执行追加,同时按分区包装返回成功或失败结果。 + +```scala +entriesPerPartition.map { case (topicIdPartition, records) => + ... + // 找到本地分区,某个分区在这个broker托管的的副本分区,可能是leader副本,也可能是follower副本 + val partition = getPartitionOrException(topicIdPartition) + // 进入分区级写入,继续检查 leader、本地日志和 min ISR。注意此时并没有判断到哪个partition是否是leader,判断逻辑在下一层 + val info = partition.appendRecordsToLeader(records, origin, requiredAcks, requestLocal, + verificationGuards.getOrElse(topicIdPartition.topicPartition(), VerificationGuard.SENTINEL), transactionVersion) + ... +} +``` + +这个方法其实算一个过渡方法,核心逻辑都委托给了appendRecordsToLeader()执行。 + +## Partition +### Partition.appendRecordsToLeader() +确认当前 broker 持有该 partition 的 leader replica,然后把 records 以 leader 身份写入本地日志。 + +```scala +def leaderLogIfLocal: Option[UnifiedLog] = { + log.filter(_ => isLeader) +} + +def isLeader: Boolean = leaderReplicaIdOpt.contains(localBrokerId) +``` + + + +```scala +val (info, leaderHWIncremented) = inReadLock(leaderIsrUpdateLock, () => { + // 判断分区是否是leader + leaderLogIfLocal match { + case Some(leaderLog) => + val minIsr = effectiveMinIsr(leaderLog) + val inSyncSize = partitionState.isr.size + + // requiredAcks == -1 表示acks=all + // acks=all 时必须满足 min.insync.replicas,否则写入即使成功也无法达到复制安全语义。 + if (inSyncSize < minIsr && requiredAcks == -1) { + throw new NotEnoughReplicasException(s"The size of the current ISR : $inSyncSize " + + s"is insufficient to satisfy the min.isr requirement of $minIsr for partition $topicPartition, " + + s"live replica(s) broker.id are : $inSyncReplicaIds") + } + + // 进入逻辑日志层,以 leader 身份分配 offset/epoch 并追加到本地日志。 + val info = leaderLog.appendAsLeader(records, this.leaderEpoch, origin, requestLocal, verificationGuard, transactionVersion) + + // ISR 可能只剩 leader 自己,写入后需要尝试推进 high watermark。 + (info, maybeIncrementLeaderHW(leaderLog)) + + case None => + // 当前 broker 不是该分区 leader,不能接收 Produce 写入。 + throw new NotLeaderOrFollowerException("Leader not local for partition %s on broker %d" + .format(topicPartition, localBrokerId)) + } +}) +``` + +关键点: + +1. **加读锁** +用 `leaderIsrUpdateLock` 保护 leader/ISR 状态,避免写入时 leader 或 ISR 状态并发变化。目的就是保证:一次 leader append 过程中,leader 身份和 ISR 判断是一致的,不会被并发 leader/ISR 更新打断。 +2. **必须是本地 leader** +`leaderLogIfLocal` 有值才允许写。否则抛 `NotLeaderOrFollowerException`。 +3. **acks=all 时检查 min ISR** +只有 `requiredAcks == -1` 时检查: + +```plain +inSyncSize < minIsr +``` + +不满足就抛 `NotEnoughReplicasException`,拒绝写入。 + +4. **真正追加日志** +调用: + +```plain +leaderLog.appendAsLeader(records, leaderEpoch, origin, ...) +``` + +这里会做 leader append:分配 offset、写入 log、处理 producer/transaction 相关校验。 + +5. **尝试推进 HW** +写完后调用: + +```plain +maybeIncrementLeaderHW(leaderLog) +``` + +如果 ISR 条件允许,可能推进 high watermark。 + +6. **返回 LogAppendInfo** +最后把 HW 是否推进写进 `LogAppendInfo`: + +```plain +info.copy(if (leaderHWIncremented) LeaderHwChange.INCREASED else LeaderHwChange.SAME) +``` + + + +## UnifiedLog +### UnifiedLog.appendAsLeader() +以 leader 身份追加 records,必要时重新校验消息、分配 offset、写入 active segment,并更新幂等/事务相关状态。 + + + +```plain +validateAndAssignOffsets = origin != AppendOrigin.RAFT_LEADER +``` + +validateAndAssignOffsets=true,则以普通leader写入。false则以Raft leader写入。 + +1. 普通leader写入 + 1. UnifiedLog 会从当前 logEndOffset 开始重新分配 offset + 2. 会设置 partition leader epoch + 3. 会更新幂等 producer、事务状态、事务索引 +2. Raft leader写入 + 1. UnifiedLog 不重新分配 offset + 2. 使用 records 里已经带好的 offset + 3. 只校验这些 offset 是否能合法追加到当前日志末尾 + +原因是:Raft 日志的 offset/位置由 Raft 层控制。Raft leader 已经决定了日志条目的顺序和 offset,UnifiedLog 不能再改,否则会破坏 Raft 复制一致性。 + + + +```scala +return append(records, origin, validateAndAssignOffsets, leaderEpoch, Optional.of(requestLocal), + verificationGuard, false, RecordBatch.CURRENT_MAGIC_VALUE, transactionVersion); +``` + +调用append()执行追加核心逻辑。 + +### UnifiedLog.append() +UnifiedLog.append()是追加日志最核心的方法,leader/follower写入最终都走到这个方法里。 + + + +```scala +maybeFlushMetadataFile(); +``` + +首先会调用maybeFlushMetadataFile(),准备元数据,确保 topic metadata 已落盘。 + +**这里的元数据和KRaft里维护的有什么区别?为什么不直接用KRaft同步了的元数据呢?** + +因为KRaft 记录的是集群当前元数据,不是每个 broker 磁盘目录的历史归属。KRaft 负责说明“集群现在应该是什么”;partition.metadata 负责证明“本地磁盘这份数据实际是谁的”。Kafka 写消息前 flush partition.metadata,是为了保证只要日志目录里有真实数据,它就一定有可靠的 topicId 身份,重启后不会把旧 topic 的数据错接到新 topic 上。 + + + +```scala +LogAppendInfo appendInfo = analyzeAndValidateRecords(...) +``` + +校验 record 格式、大小、时间戳、offset 合法性,生成 append 摘要。 + + + +```scala +if (appendInfo.firstOrLastOffsetOfFirstBatch() < localLog.logEndOffset()) + throw UnexpectedAppendOffsetException +``` + +校验 offset 不能倒退,follower 使用 leader 已分配的 offset,不能覆盖本地日志尾部。 + + + +```scala +assignEpochStartOffset(...) +``` + +更新 leader epoch cache。 + + + +```scala +maybeRoll(...) +``` + +检查单次写入大小并可能 roll segment。roll segment就是当前日志段写不下或该换新文件了,Kafka 新建一个新的 log segment,后续消息写到新 segment 里。 + + + +```scala +analyzeAndValidateProducerState(...) +``` + +校验幂等/事务 producer 状态,识别重复 batch、校验 sequence/epoch、收集事务完成信息。 + + + +```scala +localLog.append(appendInfo.lastOffset(), validRecords); +``` + +真正落本地日志。 + + + +```scala +updateHighWatermarkWithLogEndOffset(); +``` + +刚写完一批消息,log end offset 变大了,所以 Kafka 顺手让 high watermark 相关的 offset metadata 跟着刷新一下,避免它还指向旧的日志位置信息。 + + + +```scala +producerStateManager.update(...) +segment.updateTxnIndex(...) +maybeIncrementFirstUnstableOffset() +``` + +更新 producer/transaction 状态。 + + + +```scala +if (localLog.unflushedMessages() >= config().flushInterval) flush(false); +``` + +如果距离上次刷盘后,已经写入的消息数达到配置阈值,就触发一次 flush,把日志相关数据从 OS page cache 刷到磁盘。所以日志在刷盘之前,都是一直保存在Page Cache中的。 + +config().flushInterval)对应配置是: + +```scala +log.flush.interval.messages +``` + +## LocalLog +把 records 追加到当前 active log segment,然后更新本地 log end offset,推进LEO。 + +代码就两步,核心流程在LogSegment.append()。 + +```plain +segments.activeSegment().append(lastOffset, records); +updateLogEndOffset(lastOffset + 1); +``` + +## LogSegment +LogSegment.append() 是 segment 级别的底层追加方法,不负责完整写入语义校验。上层 UnifiedLog 先做 offset、segment roll、producer/事务状态等处理,然后经 LocalLog.append() 调到它。 + +```scala +public void append(long largestOffset, + MemoryRecords records) throws IOException { + // 核心总结:把 records 追加到当前 segment 的 .log 文件,并按间隔维护 offset/time 索引。 + if (records.sizeInBytes() > 0) { + LOGGER.trace("Inserting {} bytes at end offset {} at position {}", + records.sizeInBytes(), largestOffset, log.sizeInBytes()); + // 记录写入前的物理位置,后续索引会指向这个 .log 文件位置。 + int physicalPosition = log.sizeInBytes(); + + // 确认最大 offset 能用当前 segment base offset 表示成相对 offset。 + ensureOffsetInRange(largestOffset); + + // 进入 FileRecords,真正把消息字节写入 .log 文件。 + long appendedBytes = log.append(records); + LOGGER.trace("Appended {} to {} at end offset {}", appendedBytes, log.file(), largestOffset); + + for (RecordBatch batch : records.batches()) { + long batchMaxTimestamp = batch.maxTimestamp(); + long batchLastOffset = batch.lastOffset(); + // 维护 segment 内最大时间戳及其 offset,供 time index 使用。 + if (batchMaxTimestamp > maxTimestampSoFar()) { + maxTimestampAndOffsetSoFar = new TimestampOffset(batchMaxTimestamp, batchLastOffset); + } + + // 累计写入字节超过索引间隔后,写一条稀疏 offset/time 索引。 + if (bytesSinceLastIndexEntry > indexIntervalBytes) { + // 将索引写进.index文件:relativeOffset -> physicalPosition + offsetIndex().append(batchLastOffset, physicalPosition); + // 将时间索引写进.timeindex文件:timestamp -> offset + timeIndex().maybeAppend(maxTimestampSoFar(), shallowOffsetOfMaxTimestampSoFar()); + bytesSinceLastIndexEntry = 0; + } + var sizeInBytes = batch.sizeInBytes(); + physicalPosition += sizeInBytes; + bytesSinceLastIndexEntry += sizeInBytes; + } + } +} +``` + +## FileRecords.append() +FileRecords.append() 就是“容量防溢出 + 把内存 record bytes 全量写入 .log 文件 + 更新文件大小计数”。 + +```scala +public int append(MemoryRecords records) throws IOException { + // 校验records大小 + if (records.sizeInBytes() > Integer.MAX_VALUE - size.get()) + throw new IllegalArgumentException("Append of size " + records.sizeInBytes() + + " bytes is too large for segment with current file position at " + size.get()); + + // 进入 MemoryRecords,把底层 ByteBuffer 内容完整写入文件通道。 + int written = records.writeFullyTo(channel); + size.getAndAdd(written); + return written; +} +``` + +此方法不负责刷盘。append() 只是写到 FileChannel,真正 fsync 是 flush() 里的 channel.force(true)。 + +## MemoryRecords.writeFullyTo(FileChannel) +循环把底层 ByteBuffer 写入 channel,直到本批 records 的所有字节都写完。 + +核心逻辑: + +```scala +public int writeFullyTo(GatheringByteChannel channel) throws IOException { + buffer.mark(); + int written = 0; + while (written < sizeInBytes()) + written += channel.write(buffer); + // 恢复 buffer 位置,避免本次写入影响后续读取或重复写入。 + buffer.reset(); + return written; +} +``` + +# 设计思想 +经过对KafkaApis.handleProduceRequest()的分析,发现最终到写入磁盘,经过了非常长的调用链路,为什么要这么设计呢? + +Kafka 这条链路看起来很深,本质是把 **一次 Produce 写入** 拆成不同层级的职责,每层只处理自己能决定的事。 + +核心设计原理是:**从网络请求一路收敛到磁盘写入,越往下越接近存储细节,越往上越接近协议、权限、复制和事务语义。** + +```java +KafkaApis + 处理协议层:解析请求、权限、错误响应、限流 + +ReplicaManager + 处理副本层:leader 判断、acks、ISR、副本复制等待、延迟响应 + +Partition + 处理分区层:当前分区是不是 leader、min ISR、高水位推进 + +UnifiedLog + 处理逻辑日志层:offset 分配、record 校验、幂等、事务、LSO/HW 状态 + +LocalLog / LogSegment + 处理物理日志层:segment 选择、滚动、索引维护 + +FileRecords / MemoryRecords + 处理字节层:ByteBuffer 写入 FileChannel +``` + +Kafka 的 Produce 写入链路分层,是为了把协议处理、副本一致性、分区状态、日志语义、物理存储和字节写入解耦。这样每层职责清晰,既能保证高性能,又能支撑副本、事务、幂等、索引、恢复这些复杂能力。 + + + +它同时要处理: + +1. **协议兼容** +不同版本 Produce 请求、topicId/topicName、acks=0/1/all、错误码响应都不一样。 +2. **权限和配额** +要检查 topic 写权限、transactionalId 权限、带宽限流、请求限流。 +3. **副本一致性** +只能 leader 写;`acks=all` 要等 ISR 副本复制;ISR 不够要拒绝。 +4. **分区状态** +leader epoch、high watermark、log start offset、分区是否在线都要维护。 +5. **消息格式校验** +record batch 格式、大小、压缩、时间戳、magic version 都要校验或转换。 +6. **幂等和事务** +producerId、producerEpoch、sequence、事务状态、transaction index、LSO 都会影响是否能写。 +7. **物理存储** +segment 滚动、offset index、time index、文件大小、FileChannel 写入、flush 策略都属于底层存储问题。 + From f02d36652320c16c9bdb3dc84022d61a777659bb Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 24 Jun 2026 22:16:32 +0800 Subject: [PATCH 05/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4628f88..9af4654 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下, - kafka源码分析 - kafka版本:4.2 - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - - [kafka broker核心源码分析——生产者篇]() + - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) From 3dbae34c952357e6573c14d86a1d1150b655e4d5 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 24 Jun 2026 22:18:09 +0800 Subject: [PATCH 06/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E7=AB=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" | 2 +- ...\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" | 2 ++ ...\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" | 2 ++ ...\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" | 2 +- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" index 8556503..7d5a499 100644 --- "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -1,4 +1,4 @@ -当前分析版本是kafka最新版本 4.2(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) +当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) # kafka服务端核心逻辑 目前最新版本kafka已经抛弃了Zookeeper,而是自己维护元数据日志,存放在内部 topic: __cluster_metadata。 diff --git "a/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" index ea7feb9..67f03b8 100644 --- "a/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" +++ "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" @@ -1,3 +1,5 @@ +当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) + # KafkaRequestHandler核心流程 `KafkaRequestHandler.run()` 是在 **创建 **`**KafkaRequestHandlerPool**`** 时触发的**。 diff --git "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" index 20fbee5..02b1e0d 100644 --- "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" @@ -1,3 +1,5 @@ +当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) + # Kafka Rebalance 核心流程 `KafkaConsumer.poll()` 是消费者触发 rebalance 的主要入口。 diff --git "a/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" "b/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" index 69cc887..cefc08a 100644 --- "a/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" +++ "b/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" @@ -1,4 +1,4 @@ -当前分析版本是kafka最新版本 4.2(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) +当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) # kafka消费者核心入口poll kafka消费者核心入口在KafkaConsumer.poll(): From 501f2a4715b85fbd89037522a2d8cfa9ba8d31e8 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sun, 28 Jun 2026 21:48:02 +0800 Subject: [PATCH 07/37] fix --- .../concurent/CompletableFutureDemo.java | 13 +- .../CompletableFuturePrincipleTest.java | 159 ++++++++++++++++++ ...73\350\276\221\345\210\206\346\236\220.md" | 42 ++--- 3 files changed, 192 insertions(+), 22 deletions(-) create mode 100644 JdkLearn/src/test/java/com/learnjava/concurrent/CompletableFuturePrincipleTest.java diff --git a/JdkLearn/src/main/java/com/learnjava/concurent/CompletableFutureDemo.java b/JdkLearn/src/main/java/com/learnjava/concurent/CompletableFutureDemo.java index c47cce0..37d2b27 100644 --- a/JdkLearn/src/main/java/com/learnjava/concurent/CompletableFutureDemo.java +++ b/JdkLearn/src/main/java/com/learnjava/concurent/CompletableFutureDemo.java @@ -6,7 +6,8 @@ public class CompletableFutureDemo { public static void main(String[] args) throws Exception { CompletableFutureDemo completableFutureDemo = new CompletableFutureDemo(); - completableFutureDemo.test_completed_future(); +// completableFutureDemo.test_completed_future(); + completableFutureDemo.test_completed_thenApply(); } public void test_completed_future() throws Exception { @@ -15,4 +16,14 @@ public void test_completed_future() throws Exception { System.out.println(alreadyCompleted.get()); } + public void test_completed_thenApply() { + CompletableFuture source = new CompletableFuture<>(); + CompletableFuture next = + source.thenApply(x -> { + System.out.println(Thread.currentThread().getName()); + return x + 1; + }); + source.complete(10); + System.out.println(next.join()); // 11 + } } diff --git a/JdkLearn/src/test/java/com/learnjava/concurrent/CompletableFuturePrincipleTest.java b/JdkLearn/src/test/java/com/learnjava/concurrent/CompletableFuturePrincipleTest.java new file mode 100644 index 0000000..7cc5aee --- /dev/null +++ b/JdkLearn/src/test/java/com/learnjava/concurrent/CompletableFuturePrincipleTest.java @@ -0,0 +1,159 @@ +package com.learnjava.concurrent; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue;; + +/** + * @Author : haiyang.luo + * @Date : 2026/6/28 21:44 + * @Description : + */ +@Timeout(30) +public class CompletableFuturePrincipleTest { + + @Test + public void testCompleteTriggersDependentStageRegisteredBeforeCompletion() { + CompletableFuture source = new CompletableFuture<>(); + CompletableFuture dependent = source.thenApply(value -> value + 1); + + assertFalse(dependent.isDone()); + + assertTrue(source.complete(10)); + + assertEquals(11, dependent.join()); + assertTrue(dependent.isDone()); + } + + @Test + public void testDependentStageRegisteredAfterCompletionRunsOnCallerThread() { + CompletableFuture source = CompletableFuture.completedFuture(10); + String callerThreadName = Thread.currentThread().getName(); + AtomicReference callbackThreadName = new AtomicReference<>(); + + CompletableFuture dependent = source.thenApply(value -> { + callbackThreadName.set(Thread.currentThread().getName()); + return value + 1; + }); + + assertEquals(11, dependent.join()); + assertEquals(callerThreadName, callbackThreadName.get()); + } + + @Test + public void testAsyncDependentStageRunsOnExecutor() { + ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable); + thread.setName("completable-future-principle-test"); + return thread; + }); + try { + CompletableFuture source = new CompletableFuture<>(); + AtomicReference callbackThreadName = new AtomicReference<>(); + + CompletableFuture dependent = source.thenApplyAsync(value -> { + callbackThreadName.set(Thread.currentThread().getName()); + return value + 1; + }, executor); + + assertTrue(source.complete(10)); + + assertEquals(11, dependent.join()); + assertEquals("completable-future-principle-test", callbackThreadName.get()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testThenComposeCompletesAfterReturnedFutureCompletes() { + CompletableFuture source = new CompletableFuture<>(); + CompletableFuture inner = new CompletableFuture<>(); + + CompletableFuture composed = source.thenCompose(value -> inner.thenApply(innerValue -> value + innerValue)); + + assertTrue(source.complete(10)); + assertFalse(composed.isDone()); + + assertTrue(inner.complete(5)); + + assertEquals(15, composed.join()); + } + + @Test + public void testExceptionalCompletionSkipsThenApplyAndCanRecover() { + CompletableFuture source = new CompletableFuture<>(); + AtomicBoolean thenApplyCalled = new AtomicBoolean(false); + + CompletableFuture recovered = source + .thenApply(value -> { + thenApplyCalled.set(true); + return value + 1; + }) + .exceptionally(exception -> 0); + + RuntimeException failure = new RuntimeException("boom"); + assertTrue(source.completeExceptionally(failure)); + + assertEquals(0, recovered.join()); + assertFalse(thenApplyCalled.get()); + } + + @Test + public void testOnlyFirstCompletionWins() { + CompletableFuture future = new CompletableFuture<>(); + + assertTrue(future.complete("first")); + assertFalse(future.complete("second")); + assertFalse(future.completeExceptionally(new RuntimeException("boom"))); + + assertEquals("first", future.join()); + } + + @Test + public void testJoinWaitsForCompletion() throws Exception { + CompletableFuture future = new CompletableFuture<>(); + CountDownLatch joinStarted = new CountDownLatch(1); + AtomicReference joinedValue = new AtomicReference<>(); + + Thread waitingThread = new Thread(() -> { + joinStarted.countDown(); + joinedValue.set(future.join()); + }); + waitingThread.start(); + + assertTrue(joinStarted.await(5, TimeUnit.SECONDS)); + assertFalse(future.isDone()); + + assertTrue(future.complete("done")); + + waitingThread.join(TimeUnit.SECONDS.toMillis(5)); + assertFalse(waitingThread.isAlive()); + assertEquals("done", joinedValue.get()); + } + + @Test + public void testJoinWrapsFailureInCompletionException() { + CompletableFuture future = new CompletableFuture<>(); + RuntimeException failure = new RuntimeException("boom"); + + assertTrue(future.completeExceptionally(failure)); + + CompletionException exception = assertThrows(CompletionException.class, future::join); + assertSame(failure, exception.getCause()); + } +} diff --git "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" index 02b1e0d..0faf660 100644 --- "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" @@ -3,7 +3,7 @@ # Kafka Rebalance 核心流程 `KafkaConsumer.poll()` 是消费者触发 rebalance 的主要入口。 -位置在 [KafkaConsumer.java (line 916)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java:916): +位置在KafkaConsumer.java: ```plain public ConsumerRecords poll(final Duration timeout) { @@ -11,7 +11,7 @@ public ConsumerRecords poll(final Duration timeout) { } ``` -在 classic consumer 实现里,会进入 [ClassicKafkaConsumer.poll() (line 641)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ClassicKafkaConsumer.java:641): +在 classic consumer 实现里,会进入 ClassicKafkaConsumer.poll(): ```plain KafkaConsumer.poll @@ -33,7 +33,7 @@ KafkaConsumer.poll + **拉取并返回消息** 从 fetch buffer 取数据,或者发送新的 FetchRequest,并把拿到的 records 返回给用户。 -对应 [ClassicKafkaConsumer.java (line 665)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ClassicKafkaConsumer.java:665): +对应 ClassicKafkaConsumer.java: ```plain updateAssignmentMetadataIfNeeded(timer, false) @@ -41,14 +41,14 @@ pollForFetches(timer) sendFetches() ``` -再精简一句:**rebalance 不是后台自动完整完成的,classic consumer 的 group rebalance 主要是在用户线程调用 **`poll()`** 时被推进的。** + **rebalance 不是后台自动完整完成的,classic consumer 的 group rebalance 主要是在用户线程调用 **`poll()`** 时被推进的。** --- ## ConsumerCoordinator.poll() `ConsumerCoordinator.poll()` 是 consumer 端消费组协调的入口。 -位置在 [ConsumerCoordinator.java (line 530)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:530): +位置在 ConsumerCoordinator.java: ```plain public boolean poll(Timer timer, boolean waitForJoinGroup) @@ -105,7 +105,7 @@ Kafka 不希望在内部网络处理路径里直接跑用户 callback,因为 ### rejoinNeededOrPending() -这个方法命名就很有意思“是否需要重新加入消费组,或者重新加入消费组的流程仍在进行中”。首先来看请求第一次进来。 + 这个方法命名就很有意思“是否需要重新加入消费组,或者重新加入消费组的流程仍在进行中”。首先来看请求第一次进来。 核心链路: @@ -127,7 +127,7 @@ if (!subscriptions.hasAutoAssignedPartitions()) -上次 assignment 使用的 metadata 和当前 metadata 不匹配,或者说集群 topic/partition 等元数据变化,需要重新分配。 + 上次 assignment 使用的 metadata 和当前 metadata 不匹配,或者说集群 topic/partition 等元数据变化,需要重新分配。 ```java if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) { @@ -138,7 +138,7 @@ if (assignmentSnapshot != null && !assignmentSnapshot.matches(metadataSnapshot)) -上次 JoinGroup 提交的 subscription 和当前 subscription 不一致,或者说用户订阅或正则匹配结果变化,需要重新入组。 + 上次 JoinGroup 提交的 subscription 和当前 subscription 不一致,或者说用户订阅或正则匹配结果变化,需要重新入组。 ```java if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subscription())) { @@ -149,7 +149,7 @@ if (joinedSubscription != null && !joinedSubscription.equals(subscriptions.subsc -这里两处判断为啥都是上次的JoinGroup?因为这是在构造这一次JoinGroup之前,所以得那上次的JoinGroup的元数据和订阅信息和当前准备构造JoinGroup的数据进行对比。 + 这里两处判断为啥都是上次的JoinGroup?因为这是在构造当前这次JoinGroup之前的判断逻辑,所以得拿上次的JoinGroup的元数据和订阅信息和当前准备构造JoinGroup的数据进行对比。 @@ -179,7 +179,7 @@ public synchronized void requestRejoin(final String shortReason, ## Rebalance触发条件? 是否需要加入或重新加入消费组,主要由 `rejoinNeededOrPending()` 判断。 -位置在 [ConsumerCoordinator.java (line 953)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:953): +位置在 ConsumerCoordinator.java: ```plain public boolean rejoinNeededOrPending() @@ -203,7 +203,7 @@ topic 新增分区、删除分区、正则订阅匹配结果变化等。 ## ensureActiveGroup() `ensureActiveGroup()` 是“确保消费组已经可用”的核心方法。 -位置在 [AbstractCoordinator.java (line 422)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:422): +位置在 AbstractCoordinator.java: ```plain boolean ensureActiveGroup(final Timer timer) { @@ -240,7 +240,7 @@ ensureActiveGroup ## joinGroupIfNeeded() `joinGroupIfNeeded()` 是 consumer 端 rebalance 状态机的核心。 -位置在 [AbstractCoordinator.java (line 476)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:476): +位置在 AbstractCoordinator.java: ```plain boolean joinGroupIfNeeded(final Timer timer) @@ -280,7 +280,7 @@ SyncGroup 成功后,调用 `onJoinComplete(...)`,更新本地分配并触发 ## JoinGroup 请求携带什么 JoinGroup 请求里最关键的是 `metadata()` 生成的 subscription metadata。 -位置在 [ConsumerCoordinator.java (line 303)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:303): +位置在 ConsumerCoordinator.java: ```plain protected JoinGroupRequestProtocolCollection metadata() @@ -313,7 +313,7 @@ Subscription subscription = new Subscription( ## Broker 侧 JoinGroup / SyncGroup broker 网络入口仍然是 `KafkaApis.handle()`。 -位置在 [KafkaApis.scala (line 180)](/Users/apple/Documents/git_code_pr/kafka/core/src/main/scala/kafka/server/KafkaApis.scala:180): +位置在 KafkaApis.scala: ```plain case ApiKeys.JOIN_GROUP => handleJoinGroupRequest(request, requestLocal).exceptionally(handleError) @@ -322,7 +322,7 @@ case ApiKeys.SYNC_GROUP => handleSyncGroupRequest(request, requestLocal).excepti `handleJoinGroupRequest()` 会做 group 权限校验,然后交给 `groupCoordinator.joinGroup()`。 -位置在 [KafkaApis.scala (line 1381)](/Users/apple/Documents/git_code_pr/kafka/core/src/main/scala/kafka/server/KafkaApis.scala:1381): +位置在 KafkaApis.scala: ```plain groupCoordinator.joinGroup( @@ -334,7 +334,7 @@ groupCoordinator.joinGroup( `GroupCoordinatorService.joinGroup()` 最终调度 classic group 的 join 操作。 -位置在 [GroupCoordinatorService.java (line 1053)](/Users/apple/Documents/git_code_pr/kafka/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java:1053): +位置在 GroupCoordinatorService.java : ```plain runtime.scheduleWriteOperation( @@ -346,7 +346,7 @@ runtime.scheduleWriteOperation( `syncGroup()` 类似,会调度 classic group sync。 -位置在 [GroupCoordinatorService.java (line 1125)](/Users/apple/Documents/git_code_pr/kafka/group-coordinator/src/main/java/org/apache/kafka/coordinator/group/GroupCoordinatorService.java:1125): +位置在 GroupCoordinatorService.java: ```plain runtime.scheduleWriteOperation( @@ -377,7 +377,7 @@ JoinGroupResponseHandler.handle -> ConsumerCoordinator.onLeaderElected(...) ``` -位置在 [AbstractCoordinator.java (line 724)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:724): +位置在 AbstractCoordinator.java: ```plain if (joinResponse.isLeader()) { @@ -387,7 +387,7 @@ if (joinResponse.isLeader()) { } ``` -leader 的核心逻辑在 [ConsumerCoordinator.onLeaderElected() (line 686)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:686): +leader 的核心逻辑在 ConsumerCoordinator.onLeaderElected(): ```plain protected Map onLeaderElected(...) @@ -408,14 +408,14 @@ protected Map onLeaderElected(...) + **序列化 assignment** 把每个 member 的 assignment 序列化,准备放进 leader SyncGroup。 -核心代码在 [ConsumerCoordinator.java (line 724)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java:724): +核心代码在 ConsumerCoordinator.java: ```plain Map assignments = assignor.assign(metadata.fetch(), new GroupSubscription(subscriptions)).groupAssignment(); ``` -leader 计算完 assignment 后,会在 [AbstractCoordinator.java (line 836)](/Users/apple/Documents/git_code_pr/kafka/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java:836) 构造 leader SyncGroup: +leader 计算完 assignment 后,会在 AbstractCoordinator.java 构造 leader SyncGroup: ```plain SyncGroupRequest.Builder requestBuilder = From 05c4bfd3e1505268980c4e38c12171851fc351af Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 30 Jun 2026 07:13:22 +0800 Subject: [PATCH 08/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0ThreadLocal=E5=88=86?= =?UTF-8?q?=E6=9E=90=E6=96=87=E7=AB=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- JdkLearn/pom.xml | 7 + .../learnjava/concurent/ThreadLocalTest.java | 145 +++- .../TransmittableThreadLocalDemo.java | 91 +++ ...245\345\255\246\344\271\240ThreadLocal.md" | 375 ----------- ...245\345\210\206\346\236\220ThreadLocal.md" | 626 ++++++++++++++++++ 5 files changed, 863 insertions(+), 381 deletions(-) create mode 100644 JdkLearn/src/main/java/com/learnjava/concurent/TransmittableThreadLocalDemo.java delete mode 100644 "note/JDK/\344\270\200\347\257\207\346\226\207\347\253\240\345\277\253\351\200\237\346\267\261\345\205\245\345\255\246\344\271\240ThreadLocal.md" create mode 100644 "note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" diff --git a/JdkLearn/pom.xml b/JdkLearn/pom.xml index 9fb74db..0f709d3 100644 --- a/JdkLearn/pom.xml +++ b/JdkLearn/pom.xml @@ -32,6 +32,13 @@ netty-all + + + com.alibaba + transmittable-thread-local + 2.14.5 + + org.springframework.boot diff --git a/JdkLearn/src/main/java/com/learnjava/concurent/ThreadLocalTest.java b/JdkLearn/src/main/java/com/learnjava/concurent/ThreadLocalTest.java index f5103a1..f5c3a7b 100644 --- a/JdkLearn/src/main/java/com/learnjava/concurent/ThreadLocalTest.java +++ b/JdkLearn/src/main/java/com/learnjava/concurent/ThreadLocalTest.java @@ -1,9 +1,25 @@ package com.learnjava.concurent; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class ThreadLocalTest { public static void main(String[] args) throws InterruptedException { +// testThreadIsolation(); +// testInitialValue(); +// testSetGetRemove(); +// testThreadLocalAsMapKey(); +// testThreadPoolShouldRemove(); + testHashIncrementDistribution(); + } + + /** + * 核心结果:同一个ThreadLocal对象,在不同线程中保存的是不同副本。 + */ + public static void testThreadIsolation() throws InterruptedException { + System.out.println("\n==== 1. thread isolation ===="); int threads = 3; CountDownLatch countDownLatch = new CountDownLatch(threads); @@ -20,7 +36,129 @@ public static void main(String[] args) throws InterruptedException { }, "thread - " + i).start(); } countDownLatch.await(); + } + + /** + * initialValue只会在当前线程第一次get且没有值时触发。 + */ + public static void testInitialValue() throws InterruptedException { + System.out.println("\n==== 2. initialValue per thread ===="); + ThreadLocal local = new ThreadLocal() { + @Override + protected StringBuilder initialValue() { + System.out.printf("%s init value\n", Thread.currentThread().getName()); + return new StringBuilder(Thread.currentThread().getName()); + } + }; + + Runnable task = () -> { + System.out.printf("%s first get: %s\n", Thread.currentThread().getName(), local.get()); + System.out.printf("%s second get: %s\n", Thread.currentThread().getName(), local.get()); + }; + + Thread threadA = new Thread(task, "thread-A"); + Thread threadB = new Thread(task, "thread-B"); + threadA.start(); + threadB.start(); + threadA.join(); + threadB.join(); + } + + /** + * set/get/remove是ThreadLocal最常用的生命周期。 + */ + public static void testSetGetRemove() { + System.out.println("\n==== 3. set get remove ===="); + ThreadLocal local = new ThreadLocal() { + @Override + protected String initialValue() { + return "init"; + } + }; + + System.out.println("first get: " + local.get()); + local.set("changed"); + System.out.println("after set: " + local.get()); + local.remove(); + System.out.println("after remove, get again: " + local.get()); + } + + /** + * 原理要点:值不是存在ThreadLocal对象里,而是存在当前线程的ThreadLocalMap里。 + * ThreadLocal实例本身作为key,所以同一个线程可以给不同ThreadLocal保存不同值。 + */ + public static void testThreadLocalAsMapKey() { + System.out.println("\n==== 4. ThreadLocal instance as key ===="); + ThreadLocal userLocal = new ThreadLocal(); + ThreadLocal traceLocal = new ThreadLocal(); + + userLocal.set("user-1001"); + traceLocal.set("trace-abc"); + + System.out.printf("same thread:%s, userLocal:%s, traceLocal:%s\n", + Thread.currentThread().getName(), + userLocal.get(), + traceLocal.get()); + userLocal.remove(); + traceLocal.remove(); + } + + /** + * 线程池会复用线程;如果任务结束后不remove,后续任务可能读到上一个任务遗留的值。 + */ + public static void testThreadPoolShouldRemove() throws InterruptedException { + System.out.println("\n==== 5. thread pool should remove ===="); + ThreadLocal requestIdLocal = new ThreadLocal(); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + + executorService.execute(() -> { + requestIdLocal.set("request-1"); + System.out.printf("%s set value: %s\n", + Thread.currentThread().getName(), + requestIdLocal.get()); + }); + + executorService.execute(() -> { + System.out.printf("%s read old value: %s\n", + Thread.currentThread().getName(), + requestIdLocal.get()); + requestIdLocal.remove(); + }); + + executorService.execute(() -> System.out.printf("%s after remove: %s\n", + Thread.currentThread().getName(), + requestIdLocal.get())); + + executorService.shutdown(); + executorService.awaitTermination(3, TimeUnit.SECONDS); + + } + + /** + * 验证ThreadLocal中0x61c88647这个hash增量的分布效果。 + * ThreadLocalMap长度是2的幂,定位下标时使用 hash & (len - 1)。 + */ + public static void testHashIncrementDistribution() { + System.out.println("\n==== 6. hash increment distribution ===="); + int hashIncrement = 0x61c88647; + + printHashIndexSequence(hashIncrement, 16); + printHashIndexSequence(hashIncrement, 32); + } + + private static void printHashIndexSequence(int hashIncrement, int len) { + int hash = 0; + + System.out.printf("len = %d, HASH_INCREMENT & (len - 1) = %d\n", + len, + hashIncrement & (len - 1)); + + for (int i = 0; i < len; i++) { + int index = hash & (len - 1); + System.out.printf("%2d -> hash: 0x%08x, index: %2d\n", i, hash, index); + hash += hashIncrement; + } } private static class InnerClass { @@ -54,11 +192,6 @@ public void set(String word) { } private static class Counter { - private static ThreadLocal counter = new ThreadLocal() { - @Override - protected StringBuilder initialValue() { - return new StringBuilder(); - } - }; + private static ThreadLocal counter = ThreadLocal.withInitial(StringBuilder::new); } } diff --git a/JdkLearn/src/main/java/com/learnjava/concurent/TransmittableThreadLocalDemo.java b/JdkLearn/src/main/java/com/learnjava/concurent/TransmittableThreadLocalDemo.java new file mode 100644 index 0000000..aeb182c --- /dev/null +++ b/JdkLearn/src/main/java/com/learnjava/concurent/TransmittableThreadLocalDemo.java @@ -0,0 +1,91 @@ +package com.learnjava.concurent; + +import com.alibaba.ttl.TransmittableThreadLocal; +import com.alibaba.ttl.threadpool.TtlExecutors; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +/** + * 演示 TransmittableThreadLocal 在线程池中的上下文传递。 + */ +public class TransmittableThreadLocalDemo { + + private static final ThreadLocal NORMAL_THREAD_LOCAL = new ThreadLocal<>(); + private static final InheritableThreadLocal INHERITABLE_THREAD_LOCAL = new InheritableThreadLocal<>(); + private static final TransmittableThreadLocal TTL = new TransmittableThreadLocal<>(); + + public static void main(String[] args) throws InterruptedException { + testThreadLocalInThreadPool(); + testInheritableThreadLocalInThreadPool(); + testTransmittableThreadLocalInThreadPool(); + } + + /** + * 普通 ThreadLocal 不能跨线程传递值。 + */ + private static void testThreadLocalInThreadPool() throws InterruptedException { + System.out.println("\n==== 1. normal ThreadLocal ===="); + ExecutorService executorService = Executors.newFixedThreadPool(1); + + NORMAL_THREAD_LOCAL.set("normal-context"); + executorService.execute(() -> printValue("normal", NORMAL_THREAD_LOCAL.get())); + + shutdown(executorService); + NORMAL_THREAD_LOCAL.remove(); + } + + /** + * InheritableThreadLocal 只在线程创建时传递,线程池复用线程时容易读到旧值。 + */ + private static void testInheritableThreadLocalInThreadPool() throws InterruptedException { + System.out.println("\n==== 2. InheritableThreadLocal ===="); + ExecutorService executorService = Executors.newFixedThreadPool(1); + + INHERITABLE_THREAD_LOCAL.set("parent-context-1"); + executorService.execute(() -> printValue("first task", INHERITABLE_THREAD_LOCAL.get())); + sleepQuietly(200); + + INHERITABLE_THREAD_LOCAL.set("parent-context-2"); + executorService.execute(() -> printValue("second task", INHERITABLE_THREAD_LOCAL.get())); + + shutdown(executorService); + INHERITABLE_THREAD_LOCAL.remove(); + } + + /** + * TransmittableThreadLocal 通过包装线程池,在任务提交时捕获上下文,在执行时恢复上下文。 + */ + private static void testTransmittableThreadLocalInThreadPool() throws InterruptedException { + System.out.println("\n==== 3. TransmittableThreadLocal ===="); + ExecutorService rawExecutorService = Executors.newFixedThreadPool(1); + ExecutorService ttlExecutorService = TtlExecutors.getTtlExecutorService(rawExecutorService); + + TTL.set("ttl-context-1"); + ttlExecutorService.execute(() -> printValue("first ttl task", TTL.get())); + + TTL.set("ttl-context-2"); + ttlExecutorService.execute(() -> printValue("second ttl task", TTL.get())); + + shutdown(ttlExecutorService); + TTL.remove(); + } + + private static void printValue(String scene, String value) { + System.out.printf("%s, thread=%s, value=%s%n", scene, Thread.currentThread().getName(), value); + } + + private static void shutdown(ExecutorService executorService) throws InterruptedException { + executorService.shutdown(); + executorService.awaitTermination(3, TimeUnit.SECONDS); + } + + private static void sleepQuietly(long millis) { + try { + TimeUnit.MILLISECONDS.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} diff --git "a/note/JDK/\344\270\200\347\257\207\346\226\207\347\253\240\345\277\253\351\200\237\346\267\261\345\205\245\345\255\246\344\271\240ThreadLocal.md" "b/note/JDK/\344\270\200\347\257\207\346\226\207\347\253\240\345\277\253\351\200\237\346\267\261\345\205\245\345\255\246\344\271\240ThreadLocal.md" deleted file mode 100644 index 0235a9f..0000000 --- "a/note/JDK/\344\270\200\347\257\207\346\226\207\347\253\240\345\277\253\351\200\237\346\267\261\345\205\245\345\255\246\344\271\240ThreadLocal.md" +++ /dev/null @@ -1,375 +0,0 @@ - -- [前言](#前言) -- [正文](#正文) - - [1. 适用(使用)场景](#1-适用使用场景) - - [1.1 线程资源持有(线程隔离)](#11-线程资源持有线程隔离) - - [1.2 线程资源一致性](#12-线程资源一致性) - - [1.3 线程安全](#13-线程安全) - - [1.4 分布式计算](#14-分布式计算) - - [1.5 在SqlSessionManager中的应用](#15-在sqlsessionmanager中的应用) - - [1.6 在Spring框架中的TransactionContextHolder中的应用](#16-在spring框架中的transactioncontextholder中的应用) - - [2. 源码学习](#2-源码学习) - - [2.1 ThreadLocal内部使用了哪些数据结构?](#21-threadlocal内部使用了哪些数据结构) - - [2.2 源码分析](#22-源码分析) - - [3. 总结](#3-总结) - - [3.1 ThreadLocal的适用场景](#31-threadlocal的适用场景) - - [3.2 哪些开源框架、源码使用到了ThreadLocal](#32-哪些开源框架源码使用到了threadlocal) - - [3.3 关于内存泄漏](#33-关于内存泄漏) - - [3.4 其他](#34-其他) -- [引用](#引用) - -## 前言 -最近开始空闲起来了,深入学习系列以及自我提升系列都得提上日程了。本次学习的ThreadLocal,是由JDK提供的一个用于存储每个线程本地副本信息的类,它的编写者就是著名的并发包大神Doug Lea。要想深入学习一个类,首先得先阅读它的官方类注释: - -``` - * This class provides thread-local variables. These variables differ from - * their normal counterparts in that each thread that accesses one (via its - * {@code get} or {@code set} method) has its own, independently initialized - * copy of the variable. {@code ThreadLocal} instances are typically private - * static fields in classes that wish to associate state with a thread (e.g., - * a user ID or Transaction ID). -``` - -翻译过来的意思就是: -``` -这个类用于提供线程本地变量,这些变量和普通的变量不同,因为每个线程通过访问ThreadLocal的get或者 -是set方法都会有其独立的、初始化的变量副本。ThreadLocal实例通常是希望将线程独有的状态(例如用户ID、交易ID) -线程中的私有静态字段进行关联,即将线程独有的状态存储到线程中。 -``` - -``` -*

Each thread holds an implicit reference to its copy of a thread-local - * variable as long as the thread is alive and the {@code ThreadLocal} - * instance is accessible; after a thread goes away, all of its copies of - * thread-local instances are subject to garbage collection (unless other - * references to these copies exist) -``` -这段的意思是: -``` -每个线程都会持有一个指向ThreadLocal变量的隐式引用,只要线程还没有结束,该引用就不会被GC。 -但当线程结束后并且其他地方没有对这些副本进行引用,则线程本地实例的所有副本都会被GC。 -``` - -## 正文 - -### 1. 适用(使用)场景 - -源码学习总是非常枯燥的,所以得先了解要学习的类能够做什么,适用和使用场景有哪些,这样学起来就更有目的性。经过前文对于ThreadLocal源码类注释的翻译过后,我们大致知道了ThreadLocal的作用,可以概括为两点: - -1. 用于存储线程本地的副本变量,说白了就是为了做到线程隔离。 -2. 用于确保线程安全。 - -但ThreadLocal的作用不止这两点,带着疑惑我们先看下ThreadLocal有哪些使用场景。 - -#### 1.1 线程资源持有(线程隔离) -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516100901122.png) -在WEB程序中,每个线程就是一个session,不同用户访问程序会通过不同的线程来访问,通过ThreadLocal来确保同一个线程的访问获得的用户信息都是相同的,同时也不会影响其他线程的用户信息。所以ThreadLocal可以很好的确保线程之间的隔离性。 - -#### 1.2 线程资源一致性 - -![在这里插入图片描述](https://img-blog.csdnimg.cn/2020051610161534.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -![在这里插入图片描述](https://img-blog.csdnimg.cn/2020051610243492.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -嗯.... 图片有些模糊。 - - 这个场景呢,在JDBC内部都有使用到。在JDBC内部,会通过ThreadLocal来实现 **线程资源的一致性**。我们都知道,每个HTTP请求都会在WEB程序内部生成一个线程,而每个线程去访问DB的时候,都会从连接池中获取一个Connection连接用于进行数据库交互。那么当一个HTTP请求进来,该请求在程序内部调用了不同的服务,包括搜索服务、下单服务、付款服务等,在这个调用链中每次请求一个服务都需要进行一次数据库交互,那么有一个问题就是如何确保请求过程中和数据库交互的 **事务状态一致** 的问题,如果同一个请求的调用链中connection都不同,则事务就没法控制了,因此在JDBC中通过了ThreadLocal来确保每次的请求都会和同一个connection进行一一对应,确保一次请求链中都用的同一个connection,这就是 **线程资源的一致性**。 - -#### 1.3 线程安全 -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516102700304.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -基于ThreadLocal存储在Thread中作为本地副本变量的机制,保证每个线程都可以拥有自己的上下文,确保了线程安全。相比于加锁(Synchronize、Lock),ThreadLocal的效率更高。 - -#### 1.4 分布式计算 -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516102924241.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -对于分布式计算场景中,即每个线程都计算出结果后,最终通过将ThreadLocal存储的结果取出,并收集。 - -#### 1.5 在SqlSessionManager中的应用 -在SqlSessionManager中,对于SqlSession的存储,就是通过ThreadLocal来进行的。 -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516103223860.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -可以看到,在getConnection()的时候,实际上就是去从ThreadLocal中去获取连接—SqlSession。 -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516103238445.png) -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516103246506.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) - -#### 1.6 在Spring框架中的TransactionContextHolder中的应用 -在Spring框架中的TransactionContextHolder中,也同样使用了ThreadLocal,以一个分布式事务的业务场景来进行分析: -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516103340255.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -在淘宝APP中,需要购买某个商品,会涉及交易中台,履约中台。购买一个商品后,会在交易中台去更新订单,同时需要去履约中台进行合约签订。但如果淘宝APP回滚了,则履约中台和交易中台也需要进行业务回滚。对于分布式事务,需要有一个context,即资源上下文,用于存储用户的信息、订单的信息以及来源等,因此在Spring的TransactionContextHolder中,就通过ThreadLocal来存储context。 - -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516103356344.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -以上都是在学习慕课网“求老仙奶我不到P10”老师的《ThreadLocal讲解》的视频中总结的内容,如有侵权请联系删除。PS:老师讲解的非常好,建议小伙伴们都去观看学习一波。 - -### 2. 源码学习 -上面已经了解到ThreadLocal的适用(使用)场景了,下面就开始枯燥的源码学习了,在学习之前,我们先提出几个疑问: - -1. ThreadLocal是怎么保证了线程隔离的? -2. ThreadLocal注释中提到的隐式引用是什么?有什么作用? -3. ThreadLocal为什么要用到隐式引用?而不用强引用? -4. 据说ThreadLocal会发生内存泄漏?什么情况下会发生内存泄漏?如何避免内存泄漏? -5. 使用ThreadLocal有什么需要注意的点? - -#### 2.1 ThreadLocal内部使用了哪些数据结构? -首先,我们来看下ThreadLocal中几个比较重要的数据结构。 - -```Java -/** - * 用于ThreadLocal内部ThreadLocalMap数据结构的哈希值,用于降低哈希冲突。 - */ -private final int threadLocalHashCode = nextHashCode(); - -/** - * 原子操作生成哈希值,初始值为0. - */ -private static AtomicInteger nextHashCode = new AtomicInteger(); - -/* - * 用于进行计算出threadLocalHashCode的哈希值。 - */ -private static final int HASH_INCREMENT = 0x61c88647; - -/** - * 返回下一个哈希值,让哈希值散列更均匀。 - */ -private static int nextHashCode() { - return nextHashCode.getAndAdd(HASH_INCREMENT); -} -``` - - -下面将是ThreadLocal最终要的一个数据结构:ThreadLocalMap - -```Java -/** - * ThreadLocalMap其实就是一个用于ThreadLocal的自定义HashMap,它和HashMap很像。在其内部有一个自定义的Entry类, - * 并且有一个Entry数组来存储这个类的实例对象。类似于HashMap,ThreadLocalMap同样的拥有初始大小,拥有扩容阈值。 - */ -static class ThreadLocalMap { - /* - * 可以看到,Entry类继承了WeakReference类,它的含义是弱引用,即JVM进行GC时,无论当前内存是否够用, - * 都会把被WeakReference指向的对象回收掉。 - */ - static class Entry extends WeakReference> { - /** The value associated with this ThreadLocal. */ - Object value; - - Entry(ThreadLocal k, Object v) { - super(k); - value = v; - } - } - // ThreadLocalMap的初始大小 - private static final int INITIAL_CAPACITY = 16 - - // 用于存储Entry的数组 - private Entry[] table; - - private int size = 0; - - // 扩容阈值,扩容阈值为初始大小值的三分之二。 - private int threshold; // Default to 0 - - private void setThreshold(int len) { - threshold = len * 2 / 3; - } - - private static int nextIndex(int i, int len) { - return ((i + 1 < len) ? i + 1 : 0); - } - - private static int prevIndex(int i, int len) { - return ((i - 1 >= 0) ? i - 1 : len - 1); - } -} -``` - -那么对于ThreadLocalMap中,Entry为什么要继承WeakReference,而不是其他的Reference?这里由于篇幅原因,就不加以介绍,并且网上已经有很多优秀的分析博文,可以看下末文的引用,这里就不继续深入了,简单总结以下几点原因: -1. 是为了再Thread线程在执行过程中,key能够被GC掉,从而在需要彻底GC掉ThreadLocalMap时,只需要调用ThreadLocal的remove方法即可。 -2. 如果是用的强引用,虽然Entry到Thread不可达,但是和Value还有强引用的关系,是可达的,所以无法被GC掉。 - -虽然Entry使用的是WeakReference虚引用,但JVM只是回收掉了ThreadLocalMap中的key,但是value和key是强引用的(value也会引用null),所以value是无法被回收的,所以如果线程执行时间非常长,value持续不GC,就有内存溢出的风险。所以最好的做法就是调用ThreadLocal的remove方法,把ThreadLocal.ThreadLocalMap给清除掉。 - - - -#### 2.2 源码分析 - -先看下Thread类的源码,在Thread类中,定义了两个ThreadLocalMap变量 -![在这里插入图片描述](https://img-blog.csdnimg.cn/20200516131832553.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0NvZGVyQnJ1aXM=,size_16,color_FFFFFF,t_70) -这里就可以发现,ThreadLocalMap变量定义在Thread中,因而每个Thread都拥有自己的ThreadLocalMap变量,互不影响,因而实现了线程隔离性。 - -这里有一个inheritableThreadLocals,作用是用于父子线程间ThreadLocal变量的传递。详细的关于inheritableThreadLocals的分析可以学习下博文[InheritableThreadLocal详解](https://www.jianshu.com/p/94ba4a918ff5)。 - -下面回到关于ThreadLocal源码的介绍,先看看set()和get()方法源码: - -```Java - // ThreadLocal中的set()方法 - public void set(T value) { - Thread t = Thread.currentThread(); - ThreadLocalMap map = getMap(t); - if (map != null) - map.set(this, value); - else - // 将当前线程传入,作为ThreadLocalMap的引用,创建出ThreadLocalMap - createMap(t, value); - } - - // ThreadLocalMap中的set()方法 - private void set(ThreadLocal key, Object value) { - // 初始化Entry数组 - Entry[] tab = table; - int len = tab.length; - // 通过取模计算出索引值 - int i = key.threadLocalHashCode & (len-1); - - // 如果ThreadLocalMap中tab的槽位已经被使用了,则寻找下一个索引位,i=nextIndex(i, len) - for (Entry e = tab[i]; - e != null; - e = tab[i = nextIndex(i, len)]) { - ThreadLocal k = e.get(); - - if (k == key) { - e.value = value; - return; - } - // 如果key引用被回收了,则用新的key-value来替换,并且删除无用的Entry - if (k == null) { - replaceStaleEntry(key, value, i); - return; - } - } - - tab[i] = new Entry(key, value); - int sz = ++size; - // 清楚哪些get()为空的对象,然后进行rehash。 - if (!cleanSomeSlots(i, sz) && sz >= threshold) - rehash(); - } -``` - - - -```Java - public T get() { - // 获取当前线程 - Thread t = Thread.currentThread(); - // 获取线程t中的ThreadLocalMap - ThreadLocalMap map = getMap(t); - if (map != null) { - ThreadLocalMap.Entry e = map.getEntry(this); - if (e != null) { - @SuppressWarnings("unchecked") - T result = (T)e.value; - return result; - } - } - // 如果没有获取到ThreadLocalMap,则初始化一个ThreadLocalMap - return setInitialValue(); - } - ThreadLocalMap getMap(Thread t) { - return t.threadLocals; - } - // 初始化 - private T setInitialValue() { - T value = initialValue(); - Thread t = Thread.currentThread(); - ThreadLocalMap map = getMap(t); - if (map != null) - map.set(this, value); - else - // 把线程存放到当前线程的ThreadLocalMap中 - createMap(t, value); - return value; - } -``` - -知道怎么存储以及获取ThreadLocal之后,还要知道怎么清除ThreadLocal,防止内存泄漏,下面看下remove()源码: - -```Java - // ThreadLocal的remove()方法 - public void remove() { - // 获取当前线程中的ThreadLocalMap - ThreadLocalMap m = getMap(Thread.currentThread()); - if (m != null) - m.remove(this); - } - - // ThreadLocalMap中的remove()方法 - private void remove(ThreadLocal key) { - Entry[] tab = table; - int len = tab.length; - // 通过取模获取出索引位置, - int i = key.threadLocalHashCode & (len-1); - for (Entry e = tab[i]; - e != null; - e = tab[i = nextIndex(i, len)]) { - if (e.get() == key) { - e.clear(); - - expungeStaleEntry(i); - return; - } - } - } - - /** - * 清除没用的槽位以及null插槽,并且对其进行重新散列。 - */ - private int expungeStaleEntry(int staleSlot) { - Entry[] tab = table; - int len = tab.length; - - // 将插槽位置的键和值都设置为null - tab[staleSlot].value = null; - tab[staleSlot] = null; - size--; - - // 遇到null的插槽,重新散列计算哈希值。 - Entry e; - int i; - for (i = nextIndex(staleSlot, len); - (e = tab[i]) != null; - i = nextIndex(i, len)) { - ThreadLocal k = e.get(); - if (k == null) { - e.value = null; - tab[i] = null; - size--; - } else { - int h = k.threadLocalHashCode & (len - 1); - if (h != i) { - tab[i] = null; - - // Unlike Knuth 6.4 Algorithm R, we must scan until - // null because multiple entries could have been stale. - while (tab[h] != null) - h = nextIndex(h, len); - tab[h] = e; - } - } - } - return i; - } -``` - - -## 3. 总结 - -整片文章,先介绍了ThreadLocal的适用场景,然后再由此带着疑问深入学习了ThreadLocal源码,不过ThreadLocal源码中,其实还有许多没有挖掘完的细节,这部分接下来会持续的深入分析并学习,然后再过来跟下本篇博文。 - -### 3.1 ThreadLocal的适用场景 -① 线程资源持有(线程隔离) -② 线程资源一致性 -③ 线程安全 -④ 分布式计算 - -### 3.2 哪些开源框架、源码使用到了ThreadLocal -① JDBC获取Connection相关源码 -② MyBatis中的SqlSessionManager相关源码 -③ Spring框架中的TransactionContextHolder相关源码 - -### 3.3 关于内存泄漏 -由于ThreadLocalMap的Entry继承了WeakReference,所以只要JVM发起了GC,就会回收掉Entry的键,导致当线程持续运行时,ThreadLocal中value值增多,并且没法对其进行GC,所以导致内存泄漏,因此需要调用其remove方法,避免内存泄漏。 - -### 3.4 其他 -有哪里分析总结不对的地方,欢迎各位读者及时指出。 - -## 引用 -1. [求老仙奶我不到P10 老师的ThreadLocal讲解](https://www.imooc.com/video/21060) -2. [ThreadLocal分析其弱引用和可能引起的内存泄漏](https://www.jianshu.com/p/94de80aee1bf) -3. [InheritableThreadLocal详解](https://www.jianshu.com/p/94ba4a918ff5) \ No newline at end of file diff --git "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" new file mode 100644 index 0000000..e1fdb8f --- /dev/null +++ "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" @@ -0,0 +1,626 @@ +# 是什么?解决什么问题? +ThreadLocal是什么?ThreadLocal 是Java提供的一种“线程本地变量”机制。它解决的是:让同一个变量在不同线程中有各自独立的副本。不同线程访问**同一个ThreadLocal对象**时,拿到的值互不影响。 + + + +# ThreadLocal适用场景 +## 2.2 用户上下文场景 +这种场景在Web后端很常见:用户登录后,客户端每次请求会带上 token / session / cookie,服务端在过滤器、拦截器或网关层解析出用户信息,然后放到 ThreadLocal 里,后续业务代码就可以直接取当前用户。 + +```scala +// 过滤器 / 拦截器 +try { + User user = parseToken(request); + // 用户信息存入ThreadLocal + UserContext.CURRENT_USER.set(user); + // 执行拦截器 + chain.doFilter(request, response); +} finally { + UserContext.CURRENT_USER.remove(); +} +``` + +业务代码里直接通过这个代码取用户信息即可。 + +```scala +User user = UserContext.CURRENT_USER.get(); +``` + + + +多个线程访问的是同一个静态变量,这个静态变量指向同一个 ThreadLocal 对象,但是每个线程保存值的位置不同。 + +```plain +同一个 ThreadLocal 对象:UserContext.CURRENT_USER + +Thread-8 + └── ThreadLocalMap + └── key: UserContext.CURRENT_USER -> value: "userA" + +Thread-9 + └── ThreadLocalMap + └── key: UserContext.CURRENT_USER -> value: "userB" + +Thread-10 + └── ThreadLocalMap + └── key: UserContext.CURRENT_USER -> value: "userC" +``` + + + ++ 保存当前用户信息,比如登录用户 userId ++ 保存请求链路 ID,比如 traceId ++ 保存数据库连接、事务上下文 ++ 在线程执行过程中传递上下文参数,避免方法层层传参 + +# ThreadLocal中的核心数据结构 +ThreadLocal中核心的数据结构有3个,分别是:Thread、ThreadLocal、ThreadLocalMap。他们之间的关系如下图: + +```scala +Thread + └── ThreadLocal.ThreadLocalMap threadLocals + └── Entry[] + ├── Entry(ThreadLocal key, Object value) + ├── Entry(ThreadLocal key, Object value) + └── ... +``` + +在Thread内部,都有一个成员变量threadLocals,它是一个ThreadLocalMap类型。 + +```plain +ThreadLocal.ThreadLocalMap threadLocals = null; +``` + +这说明ThreadLocal 的数据实际是挂在 Thread 对象上的。 + +```java +/** + * ThreadLocalMap其实就是一个用于ThreadLocal的自定义HashMap,它和HashMap很像。在其内部有一个自定义的Entry类, + * 并且有一个Entry数组来存储这个类的实例对象。类似于HashMap,ThreadLocalMap同样的拥有初始大小,拥有扩容阈值。 + */ +static class ThreadLocalMap { + /* + * 可以看到,Entry类继承了WeakReference类,它的含义是弱引用,即JVM进行GC时,无论当前内存是否够用, + * 都会把被WeakReference指向的对象回收掉。 + */ + static class Entry extends WeakReference> { + /** The value associated with this ThreadLocal. */ + Object value; + + Entry(ThreadLocal k, Object v) { + super(k); + value = v; + } + } + // ThreadLocalMap的初始大小 + private static final int INITIAL_CAPACITY = 16 + + // 用于存储Entry的数组 + private Entry[] table; + + // ThreadLocalMap中的元素个数 + private int size = 0; + + // 扩容阈值,扩容阈值为初始大小值的三分之二。 + private int threshold; // Default to 0 + + private void setThreshold(int len) { + threshold = len * 2 / 3; + } + + // 在table中获取下一个索引位置,如果到达数组末尾,则从头部继续扫描,解决哈希冲突 + private static int nextIndex(int i, int len) { + return ((i + 1 < len) ? i + 1 : 0); + } + + // prevIndex() 获取环形数组中的上一个索引下标;如果到达头部 0,则从尾部 len - 1 继续。 + private static int prevIndex(int i, int len) { + return ((i - 1 >= 0) ? i - 1 : len - 1); + } +} +``` + + + + + +ThreadLoca中的核心数据结构 + +```java +// ThreadLoca中的黄金比例常量 +private static final int HASH_INCREMENT = 0x61c88647; +private final int threadLocalHashCode = nextHashCode(); + +private static int nextHashCode() { + return nextHashCode.getAndAdd(HASH_INCREMENT); +} +``` + +这几个核心变量在ThreadLocalMap.set()中有重要作用,在ThreadLocalMap.set()中进行了深入的分析。 + + + +## ThreadLocalMap.Entry +Entry是ThreadLocalMap数组中的核心元素,它继承了WeakReference。核心原因是:让 ThreadLocal 对象在外部没有强引用时,可以被 GC 回收,避免 ThreadLocal key 被线程长期强引用住。 + +### 如果Entry继承的强引用 +假设Entry强引用ThreadLocal: + +```java +Thread -> ThreadLocalMap -> Entry -> ThreadLocal +``` + +在线程池场景下,线程可能长期存活。只要线程不结束,ThreadLocalMap 就还在,Entry 也还在,那么 ThreadLocal 对象就永远无法被 GC。即使业务代码已经不再持有这个 ThreadLocal 变量了,它也会被 Entry 强行引用住。这会导致:**ThreadLocal 对象无法回收,对应的 value 也无法回收。** + +这个后果就是线程长期持有已经没用的 ThreadLocal 和 value,导致内存释放不了,严重时内存泄漏、数据串用、甚至 OOM。 + + + +### 如果Entry继承的弱引用 +现在Entry继承了弱引用,则: + +```java +Thread -> ThreadLocalMap -> Entry --weak--> ThreadLocal +``` + +当业务代码不再强引用 ThreadLocal 时,threadLocal = null。GC 可以回收这个 ThreadLocal key,然后ThreadLocalMap在set/get/remove等操作时,会顺手清理这种 stale entry。 + + + +但是现在继承了弱引用的Entry还是有内存泄漏风险,如果key=null了,但是value还是被业务对象关联,value 仍然可能被 Entry 强引用着。所以在使用ThreadLocal时,都建议: + +```java +try { + threadLocal.set(value); + // 使用 +} finally { + threadLocal.remove(); +} +``` + +# ThreadLocal中的核心方法 +## ThreadLocalMap.set() +ThreadLocalMap.set()的核心注释逻辑如下: + +```java +private void set(ThreadLocal key, Object value) { + + // 获取当前线程 ThreadLocalMap 底层的 Entry 数组 + Entry[] tab = table; + + // 获取数组长度,ThreadLocalMap 的长度始终是 2 的幂 + int len = tab.length; + + // 根据 ThreadLocal 的 hash 值计算它在数组中的初始下标 + // len - 1 用作位运算取模,等价于 hash % len + int i = key.threadLocalHashCode & (len-1); + + // 三个作用 + // 1. 找到相同 key 就覆盖 value + // 2. 遇到 key 为 null 的过期 Entry 就走替换和清理逻辑 + // 3. 不是1、2,就继续往后找,直到遇到 null 槽位退出循环 + for (Entry e = tab[i]; + e != null; + // 3. 当前槽位不匹配时,移动到下一个槽位继续查找 + e = tab[i = nextIndex(i, len)]) { + + // Entry 继承 WeakReference,这里获取弱引用中的 ThreadLocal key + // 如果 ThreadLocal 已经被 GC 回收,k 会是 null + ThreadLocal k = e.get(); + + // 1. 如果当前 Entry 的 key 就是要设置的 ThreadLocal + // 说明这个 ThreadLocal 在当前线程中已经有值了 + if (k == key) { + // 直接覆盖旧 value + e.value = value; + + // 设置完成,直接返回 + return; + } + + // 2. 如果 key 为 null,说明这是一个 stale entry + // 即 ThreadLocal 对象已经被 GC 回收,但 value 还残留在数组里 + if (k == null) { + // 用当前新 key/value 替换这个过期 Entry + // 同时会顺带清理一段连续区域里的过期 Entry + replaceStaleEntry(key, value, i); + + // 替换和清理完成,直接返回 + return; + } + } + + // 执行到这里,说明从初始位置向后探测时遇到了 null 槽位 + // 当前 ThreadLocal 在 map 中还不存在,且当前位置可以插入 + tab[i] = new Entry(key, value); + + // 元素数量加 1 + int sz = ++size; + + // 尝试清理一些过期 Entry + // 如果没有清理掉任何 Entry,并且当前元素数量达到扩容阈值,则执行 rehash + if (!cleanSomeSlots(i, sz) && sz >= threshold) + rehash(); +} +``` + +几个需要注意的地方,下面重点讲下。 + +ThreadLoca中的黄金比例常量: + +```java +private static final int HASH_INCREMENT = 0x61c88647; +private final int threadLocalHashCode = nextHashCode(); + +private static int nextHashCode() { + return nextHashCode.getAndAdd(HASH_INCREMENT); +} +``` + +它的作用就是让连续创建的 ThreadLocal 的 hash 值,在经过 hash & (len - 1) 后,落到数组下标时更均匀、更分散。 + +```java +int i = key.threadLocalHashCode & (len-1); +``` + +此处进行的是二进制运算,len初始长度是16,16-1=15,二进制也就是 1111。0x61c88647 的低四位是最后一个十六进制位:0x7,换算成二进制也就是:0111。所以 + +```java +int i = 0111 && 1111; +// i = 0111; +// i = 7; +``` + +由于nextHashCode()是累加HASH_INCREMENT,所以:int i = key.threadLocalHashCode & (len-1); 最终得到的效果就是: + +```plain +0, 7, 14, 5, 12, 3, 10, 1, 8, 15, 6, 13, 4, 11, 2, 9 +``` + +这 16 个位置刚好都走一遍,而且不是连续的0,1,2,3,4,5挨着走。 + + + +## ThreadLocalMap.replaceStaleEntry() +ThreadLocalMap.replaceStaleEntry()的核心代码注释如下: + +```java +// 在发现过期槽位 staleSlot 时,插入或更新 key/value,并清理过期 Entry +private void replaceStaleEntry(ThreadLocal key, Object value, + int staleSlot) { + // 获取 ThreadLocalMap 的底层数组 + Entry[] tab = table; + + // 获取数组长度 + int len = tab.length; + + // 临时保存当前扫描到的 Entry + Entry e; + + // 记录需要开始清理过期 Entry 的位置,默认从 staleSlot 开始 + int slotToExpunge = staleSlot; + + // 从 staleSlot 向前扫描,寻找同一连续区间里更靠前的过期槽位 + for (int i = prevIndex(staleSlot, len); + // 遇到 null 说明连续探测区间结束 + (e = tab[i]) != null; + // 继续向前扫描 + i = prevIndex(i, len)) + + // 如果 key 已被 GC,说明这是过期 Entry + if (e.get() == null) + // 更新清理起点为更靠前的过期槽位 + slotToExpunge = i; + + // 从 staleSlot 向后扫描,查找 key 是否已经存在 + for (int i = nextIndex(staleSlot, len); + // 遇到 null 说明连续探测区间结束 + (e = tab[i]) != null; + // 继续向后扫描 + i = nextIndex(i, len)) { + + // 取出 Entry 中弱引用的 ThreadLocal key + ThreadLocal k = e.get(); + + // 如果找到了当前要设置的 key + if (k == key) { + // 更新已有 Entry 的 value + e.value = value; + + // 将 staleSlot 上的过期 Entry 移到当前位置 + tab[i] = tab[staleSlot]; + + // 将更新后的有效 Entry 放到 staleSlot,尽量靠近原始 hash 位置 + tab[staleSlot] = e; + + // 如果向前没有找到过期槽位,则从当前位置开始清理 + if (slotToExpunge == staleSlot) + slotToExpunge = i; + + // 清理过期 Entry,并继续做启发式清理 + cleanSomeSlots(expungeStaleEntry(slotToExpunge), len); + + // 更新完成,结束方法 + return; + } + + // 如果当前 Entry 过期,且之前还没找到清理起点 + if (k == null && slotToExpunge == staleSlot) + + // 记录第一个向后扫描到的过期槽位 + slotToExpunge = i; + } + + // 没找到已有 key,先断开 staleSlot 中旧 value 的强引用 + tab[staleSlot].value = null; + + // 复用 staleSlot,放入新的 key/value + tab[staleSlot] = new Entry(key, value); + + // 如果 staleSlot 附近还有其他过期 Entry + if (slotToExpunge != staleSlot) + // 从记录的位置开始清理过期 Entry,并继续启发式清理 + cleanSomeSlots(expungeStaleEntry(slotToExpunge), len); +} +``` + +这个方法的核心作用就是:当 set() 过程中遇到过期 Entry 时,复用这个过期槽位保存新的 key/value,并顺带清理当前连续探测区间内的其他过期 Entry,最后维护开放地址法的查找连续性。 + + + +## ThreadLocalMap.expungeStaleEntry() +ThreadLocalMap.expungeStaleEntry()核心逻辑注释如下: + +```java +// 清理 staleSlot 位置的过期 Entry,并重新整理后续冲突链 +private int expungeStaleEntry(int staleSlot) { + // 获取底层 Entry 数组 + Entry[] tab = table; + + // 获取数组长度 + int len = tab.length; + + // 断开 staleSlot 中 value 的强引用 + tab[staleSlot].value = null; + + // 清空 staleSlot 槽位 + tab[staleSlot] = null; + + // 有效元素数量减 1 + size--; + + // 临时保存当前扫描到的 Entry + Entry e; + + // 当前扫描下标 + int i; + + // 从 staleSlot 的下一个位置开始向后扫描 + for (i = nextIndex(staleSlot, len); + // 遇到 null,说明当前连续探测区间结束 + (e = tab[i]) != null; + // 继续扫描下一个槽位 + i = nextIndex(i, len)) { + + // 获取 Entry 弱引用的 ThreadLocal key + ThreadLocal k = e.get(); + + // 如果 key 已被 GC,说明当前 Entry 也是过期 Entry + if (k == null) { + // 断开 value 强引用 + e.value = null; + + // 清空当前槽位 + tab[i] = null; + + // 有效元素数量减 1 + size--; + + // 如果当前 Entry 仍然有效 + } else { + // 重新计算该 key 理论上应该落到的初始槽位 + int h = k.threadLocalHashCode & (len - 1); + + // 如果当前位置不是它理论上的初始槽位 + if (h != i) { + // 先清空当前位置,准备重新放置 + tab[i] = null; + + // 从理论槽位 h 开始,向后找第一个空槽位 + while (tab[h] != null) + + // 遇到占用则继续向后线性探测 + h = nextIndex(h, len); + + // 将 Entry 移动到更合适的位置 + tab[h] = e; + } + } + } + + // 返回遇到的 null 槽位下标,也就是清理结束位置 + return i; +} +``` + +这个方法核心作用就是:清理一个连续探测区间里的过期 Entry,并对还有效的 Entry 重新哈希摆放,避免清理产生的空洞破坏后续查找。 + + + +## ThreadLocalMap.cleanSomeSlots() +ThreadLocalMap.cleanSomeSlots()核心逻辑注释如下: + +```java +// 启发式清理一部分过期 Entry,避免每次都全表扫描 +private boolean cleanSomeSlots(int i, int n) { + // 标记本次是否清理过过期 Entry + boolean removed = false; + + // 获取底层 Entry 数组 + Entry[] tab = table; + + // 获取数组长度 + int len = tab.length; + + // 至少执行一次扫描 + do { + // 从传入位置的下一个槽位开始检查 + i = nextIndex(i, len); + + // 获取当前槽位的 Entry + Entry e = tab[i]; + + // 如果槽位不为空,并且 key 已被 GC,说明是过期 Entry + if (e != null && e.get() == null) { + // 发现过期 Entry 后,把扫描预算扩大到整张表长度 + n = len; + + // 标记已经发生清理 + removed = true; + + // 从当前位置开始清理过期 Entry,并返回清理结束后的 null 槽位位置 + i = expungeStaleEntry(i); + } + + // 每轮把扫描预算减半;n 变成 0 时停止 + // 每次把 n 无符号右移一位,也就是约等于除以 2。 + } while ( (n >>>= 1) != 0); + + // 返回本次是否清理过过期 Entry + return removed; +} +``` + +这个方法的核心作用是:cleanSomeSlots 是 ThreadLocalMap 的“低成本局部清理机制”:平时少扫一点,发现过期 Entry 后多扫一点,用来控制内存泄漏风险和哈希表性能退化。 + + + +# InheritableThreadLocal +InheritableThreadLocal的作用场景是,父线程里设置了线程本地变量,希望新创建的子线程能自动拿到一份初始值。 + +场景业务场景包括: + +1)传递请求上下文:比如 traceId、requestId、租户 ID、用户信息等。 + +2)父子线程任务链路追踪:日志 MDC、分布式链路追踪中可能需要把父线程上下文传给子线程。 + +普通 ThreadLocal 是线程隔离的,子线程拿不到父线程的值: + +```java +ThreadLocal local = new ThreadLocal<>(); + +local.set("parent"); + +new Thread(() -> { + System.out.println(local.get()); // null +}).start(); +``` + + 而 InheritableThreadLocal 可以让子线程继承父线程的值: + +```java +InheritableThreadLocal local = new InheritableThreadLocal<>(); + +local.set("parent"); + +new Thread(() -> { + System.out.println(local.get()); // parent +}).start(); +``` + + + +核心传值逻辑在Thread.init()方法中。 + +```java +private void init(ThreadGroup g, Runnable target, String name, + long stackSize, AccessControlContext acc, + boolean inheritThreadLocals) { + ... + // 创建子线程时复制 + if (inheritThreadLocals && parent.inheritableThreadLocals != null) + this.inheritableThreadLocals = + ThreadLocal.createInheritedMap(parent.inheritableThreadLocals); + ... +} + +static ThreadLocalMap createInheritedMap(ThreadLocalMap parentMap) { +return new ThreadLocalMap(parentMap); +} + +private ThreadLocalMap(ThreadLocalMap parentMap) { + Entry[] parentTable = parentMap.table; + int len = parentTable.length; + setThreshold(len); + table = new Entry[len]; + + // 拷贝父线程的entry + for (int j = 0; j < len; j++) { + Entry e = parentTable[j]; + if (e != null) { + @SuppressWarnings("unchecked") + ThreadLocal key = (ThreadLocal) e.get(); + if (key != null) { + Object value = key.childValue(e.value); + Entry c = new Entry(key, value); + int h = key.threadLocalHashCode & (len - 1); + while (table[h] != null) + h = nextIndex(h, len); + table[h] = c; + size++; + } + } + } +} +``` + + + +但是现在基本都没有直接通过new Thread()的方式创建线程了,基本都是通过线程池来管理线程。而在常规业务线程池里,InheritableThreadLocal 基本不适合作为上下文传递方案。它的继承时机是**“创建线程时”**,而线程池的线程通常早就创建好了,任务提交时不会重新继承父线程上下文。 + +线程池上下文传递方案,用的最多的就是阿里的TransmittableThreadLocal,简称 TTL。 + + + +# TransmittableThreadLocal +TransmittableThreadLocal是怎么实现的线程池上下文传递方案?它核心原理就是:在提交任务时捕获父线程上下文,在任务执行时恢复到工作线程,执行完再还原工作线程原来的上下文。 + + + +InheritableThreadLocal在线程池场景最核心的原因,就是当提交任务到线程池,工作线程已经存在,不会重新new Thread(),所以不会重新复制inheritableThreadLocals。 + +TransmittableThreadLocal底层核心伪代码如下。 + +```java +class TtlRunnable implements Runnable { + private final Runnable runnable; + private final Object captured; + + TtlRunnable(Runnable runnable) { + this.runnable = runnable; + + // 1. 在提交任务的线程中捕获上下文 + this.captured = Transmitter.capture(); + } + + @Override + public void run() { + // 2. 在工作线程执行前,把捕获的上下文恢复到当前线程 + // 也就是将父线程的值传到当前线程池要执行的线程 + Object backup = Transmitter.replay(captured); + + try { + // 3. 执行业务逻辑 + runnable.run(); + } finally { + // 4. 执行完成后,恢复工作线程原来的上下文,避免下次线程池使用的线程时出现值污染 + Transmitter.restore(backup); + } + } +} +``` + + + +总结:TransmittableThreadLocal在线程池里传值,是通过包装任务,在任务提交时捕获父线程的 TTL 快照,在工作线程执行前恢复这份快照,执行结束后再还原工作线程原上下文来实现的。 + From e0bb50f8ae9de2d1e252ebacba83f7a75810f4a0 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 30 Jun 2026 07:15:03 +0800 Subject: [PATCH 09/37] update --- README.md | 2 +- ...6\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9af4654..255ed3d 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - [深入学习String源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) - [深入学习String源码与底层(二)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%BA%8C%EF%BC%89.md) - [深入解读CompletableFuture源码与原理](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E8%AF%BBCompletableFuture%E6%BA%90%E7%A0%81%E4%B8%8E%E5%8E%9F%E7%90%86.md) - - [一篇文章快速深入学习ThreadLocal](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E4%B8%80%E7%AF%87%E6%96%87%E7%AB%A0%E5%BF%AB%E9%80%9F%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0ThreadLocal.md) + - [深入分析ThreadLocal](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%88%86%E6%9E%90ThreadLocal.md) - [深入学习Java volatile关键字](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Java%20volatile%E5%85%B3%E9%94%AE%E5%AD%97.md) - [深入学习Thread底层原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Thread%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81.md) - [深入学习JDK1.7、8 HashMap扩容原理]() diff --git "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" index e1fdb8f..59756a7 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" @@ -4,7 +4,7 @@ ThreadLocal是什么?ThreadLocal 是Java提供的一种“线程本地变量 # ThreadLocal适用场景 -## 2.2 用户上下文场景 +## 用户上下文场景 这种场景在Web后端很常见:用户登录后,客户端每次请求会带上 token / session / cookie,服务端在过滤器、拦截器或网关层解析出用户信息,然后放到 ThreadLocal 里,后续业务代码就可以直接取当前用户。 ```scala From 4604c0da0c0602998557396515e74abbf48d8b6b Mon Sep 17 00:00:00 2001 From: coderbruis Date: Thu, 2 Jul 2026 11:13:44 +0800 Subject: [PATCH 10/37] =?UTF-8?q?=E6=96=B0=E5=A2=9Ekafka=20ISR=E5=8E=9F?= =?UTF-8?q?=E7=90=86=E5=88=86=E6=9E=90:?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +- ...25\345\261\202\345\216\237\347\220\206.md" | 509 ++++++++++++++++++ ...20\347\240\201\345\210\206\346\236\220.md" | 3 +- ...37\344\272\247\350\200\205\347\257\207.md" | 3 +- ...73\350\276\221\345\210\206\346\236\220.md" | 3 +- ...20\357\274\210\344\270\200\357\274\211.md" | 3 +- 6 files changed, 524 insertions(+), 13 deletions(-) create mode 100644 "note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" diff --git a/README.md b/README.md index 255ed3d..448b518 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,13 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 # 目录 +- kafka源码分析 + - kafka版本:4.2 + - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) + - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) + - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) + - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) + - JDK源码学习 - JDK版本:1.8.0_77 - [深入学习String源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) @@ -145,15 +152,6 @@ Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下, - RocketMQ底层源码解析-本地调试RocketMQ源码 - RocketMQ底层源码解析-NameServer分析 - - -- kafka源码分析 - - kafka版本:4.2 - - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) - - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) - 持续更新中... diff --git "a/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" "b/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" new file mode 100644 index 0000000..02e5bd2 --- /dev/null +++ "b/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" @@ -0,0 +1,509 @@ ++ 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:https://github.com/coderbruis/kafka source_code_analysis分支,底层原理持续更新) ++ 转载请标明出处 + +# Kafka ISR是什么?解决什么问题? +## 是什么? +Kafka ISR是In-Sync Replicas,意思是“与leader保持同步的副本集合”。在Kafka中会有leader副本和follower副本,下面举例: + +partition-0有三个副本,replicas = [1, 2, 3],leader副本是:1,另外两个是follower副本:2, 3。 + +当网络状况良好,follower副本同步状况良好,都追上了leader副本,则ISR=[1,2,3],说明1,2,3都是同步副本。可是某一时刻3卡住、宕机、网络慢,长时间追不上leader,则此时ISR=[1,2]。还有一种最差的情况就是2、3网络都慢,导致最后ISR=[1],ISR就只剩下leader副本了。这里需要注意,leader副本一直都是在ISR中的。 + +## 解决什么问题? +ISR核心为了解决两个问题: + + 1. 判断消息是否足够安全。 + 2. 选 leader 时尽量不丢数据。 + +### 判断消息是否足够安全 +当producer配置:acks=all。Kafka不是等所有replicas都写成功,而是等当前 ISR 里的副本都确认。例如: + +replicas = [1, 2, 3],ISR = [1, 2],此时acks=all只需要leader-1和follower-2写成功,不会等已经落后的follower-3,**这样避免一个慢副本拖垮整个消息写入。** + +### 选 leader 时尽量不丢数据 +leader挂了以后,Kafka优先从ISR里选新leader。因为ISR里的副本代表“最近是跟上leader的”,更可能拥有已提交的数据。如果选择了非ISR中的副本,则说明可能是缺数据的,非ISR副本当了leader,可能会丢消息。 + +# Kafka ISR 底层原理与源码链路 +已经知道了Kafka ISR的基本概念以及解决什么问题,下文梳理Kafka ISR从初始化、运行时变更、controller 提交确认,到 broker 应用 metadata 的完整链路。 + +## Kafka中ISR的两层状态 +在Kafka中,ISR有两层状态: + + - controlle权威状态:PartitionRegistration.isr + - leader broker运行时状态:Partition.partitionState.isr + +ISR 的初始化以及变更都是是 controller metadata log中实现,leader broker只能根据 follower 复制进度提出 ISR 变更,真正生效必须经过 controller 校验并写入 metadata log。 + +kafka的ISR是controller说了算,也就是KRaft的controller leader最终控制的。 + +## Controller Leader 初始化ISR +ISR 的初始值是在controller创建topic/partition时生成的 + +源码链路: + +```latex +ReplicationControlManager.createTopics() + -> createTopic() + -> 生成每个 partition 的 replica assignment + -> 过滤出 active replicas 作为初始 ISR + -> buildPartitionRegistration(...) + -> PartitionRegistration.isr + -> PartitionRegistration.toRecord(...) + -> PartitionRecord 写入 metadata log +``` + +关键逻辑在 ReplicationControlManager.createTopic(): + +```latex +List isr = partitionAssignment.replicas().stream() + .filter(clusterControl::isActive) + .toList(); +``` + +这里的含义是:创建 topic 时,controller 会先确定每个 partition 的副本列表 replicas,然后从这些副本里筛出当前可用的 broker,也就是 active replicas,作为这个 partition 的初始 ISR。 + +然后 controller 用这些信息构造 PartitionRegistration: + +```latex +return new PartitionRegistration.Builder() + .setReplicas(...) + .setIsr(...) + .setLeader(isr.get(0)) + .setLeaderEpoch(0) + .setPartitionEpoch(0) + .build(); +``` + +这段源码有几个需要注意的点: + + - replicas:这个 partition 的完整副本集合。 + - isr:创建时处于 active 状态的副本集合。 + - leader:默认取isr.get(0),也就是初始 ISR 里的第一个副本。 + + + +PartitionRegistration 会被转成 metadata record: + +```latex +records.add(info.toRecord(topicId, partitionIndex, ...)); +``` + +随后在QuorumController.ControllerWriteEvent.run()中被追加到metadata log中。 + +```latex +long lastOffset = raftClient.prepareAppend(controllerEpoch, records); +``` + +## Broker 如何应用初始 ISR +Broker 不会自己生成初始 ISR,它是从 metadata log 里读取 controller 写入的PartitionRecord,然后应用到本地分区状态中。 + +源码链路: + +```latex +metadata log 中的 PartitionRecord + -> MetadataDelta.replay() + -> TopicsDelta.replay() + -> TopicDelta.replay(PartitionRecord) + -> new PartitionRegistration(record) + -> MetadataImage / TopicsImage 中保存 PartitionRegistration.isr + -> BrokerMetadataPublisher.onMetadataUpdate() + -> ReplicaManager.applyDelta() + -> applyLocalLeadersDelta() / applyLocalFollowersDelta() + -> Partition.makeLeader() / Partition.makeFollower() + -> Partition.updateAssignmentAndIsr() + -> Partition.partitionState.isr +``` + +关键逻辑在 TopicDelta.replay(PartitionRecord): + +```latex +partitionChanges.put(record.partitionId(), new PartitionRegistration(record)); +``` + +这里的含义是:broker 消费 metadata log 时,遇到 PartitionRecord,会把 record 反序列化成 PartitionRegistration。而 PartitionRegistration 里面就包含 controller 初始化时写入的 ISR。 + +也就是说,controller 写入的是:PartitionRecord.isr,broker读取后恢复成:PartitionRegistration.isr。 + +然后 broker 侧的 metadata publisher 会把新的 metadata image 应用到运行时模块: + +```latex +replicaManager.applyDelta(topicsDelta, newImage) +``` + + + +ReplicaManager.applyDelta() 会判断当前 broker 在这些 partition 里的角色: + +```latex +如果当前 broker 是 leader + -> applyLocalLeadersDelta() + -> Partition.makeLeader() + +如果当前 broker 是 follower + -> applyLocalFollowersDelta() + -> Partition.makeFollower() +``` + +leader 场景下,核心逻辑进入 Partition.makeLeader(): + +```latex +partition.makeLeader(info.partition, isNew, offsetCheckpoints, Some(info.topicId), ...) +``` + +这里传进去的 info.partition 就是 PartitionRegistration,里面带着 controller 写入的初始 ISR。 + +随后 makeLeader() 会调用: + +```latex +updateAssignmentAndIsr(...) +``` + +把 PartitionRegistration 里的副本信息、ISR、leader epoch、partition epoch 等信息应用到 broker 本地的 Partition 对象中。 + +最终落到 broker 运行时状态里:**Partition.partitionState.isr**。后续leader副本和follower副本会用这份ISR做不通的逻辑。 + + - leader broker 会用本地 ISR 判断 acks=all、min.insync.replicas、HW 推进、ISR 扩缩容。 + - follower broker 也会应用这份 ISR,但 follower 不负责决定 ISR 变更,它主要根据 leader 信息启动 fetch,继续追 leader 数据。 + + + +## ISR 如何判断消息是否足够安全 +消息是否“足够安全”,核心不是看所有 replicas,而是看 **ISR 里的副本是否满足要求**。 + +源码链路: + +```plain +KafkaApis.handleProduceRequest() + -> replicaManager.handleProduceAppend() + -> ReplicaManager.appendRecords() + -> ReplicaManager.appendRecordsToLeader() + -> ReplicaManager.appendToLocalLog() + -> Partition.appendRecordsToLeader() + -> 判断 ISR 数量是否满足 min.insync.replicas + -> leader 本地追加消息 + -> 如果 requiredAcks = -1,进入 DelayedProduce 等待 ISR 副本追上 + -> Partition.checkEnoughReplicasReachOffset() + -> 判断 high watermark 是否达到 requiredOffset + -> 返回 produce response +``` + +关键入口在 `KafkaApis.handleProduceRequest()`: + +```plain +replicaManager.handleProduceAppend( + timeout = produceRequest.timeout.toLong, + requiredAcks = produceRequest.acks, + ... +) +``` + +这里的 `requiredAcks` 就是 producer 配置里的 `acks`。 + +常见情况: + +```plain +acks = 0 + -> broker 不需要返回确认 + +acks = 1 + -> leader 写入成功就返回成功 + +acks = -1 / all + -> leader 写入成功后,还要等 ISR 副本复制到足够位置 +``` + +真正判断 ISR 是否足够的逻辑在 `Partition.appendRecordsToLeader()`: + +```plain +val minIsr = effectiveMinIsr(leaderLog) +val inSyncSize = partitionState.isr.size + +if (inSyncSize < minIsr && requiredAcks == -1) { + throw new NotEnoughReplicasException(...) +} +``` + +这里的含义是:如果 producer 使用 `acks=all`,那么 leader 在写入消息之前,会先检查当前 ISR 数量是否小于 `min.insync.replicas`。 + +如果: + +```plain +ISR 数量 < min.insync.replicas +``` + +那么这条消息连 leader 本地都不会写入,直接返回: + +```plain +NOT_ENOUGH_REPLICAS +``` + +如果 ISR 数量满足要求,leader 才会执行本地追加: + +```plain +val info = leaderLog.appendAsLeader(...) +``` + +但是对于 `acks=all`,leader 本地写成功还不代表请求可以立刻返回成功。因为 `acks=all` 的语义是:消息需要被 ISR 中的副本复制到足够安全的位置。 + +所以写入之后,会进入 `DelayedProduce` 等待: + +```plain +ReplicaManager.maybeAddDelayedProduce(...) +``` + +判断是否需要等待的条件是: + +```plain +requiredAcks == -1 +``` + +也就是只有 `acks=all` 才会走这条等待 ISR 确认的逻辑。 + + + +等待过程中,会反复检查: + +```plain +partition.checkEnoughReplicasReachOffset(requiredOffset) +``` + +关键逻辑在 `Partition.checkEnoughReplicasReachOffset()`: + +```plain +if (leaderLog.highWatermark >= requiredOffset) { + if (minIsr <= curMaximalIsr.size) + (true, Errors.NONE) + else + (true, Errors.NOT_ENOUGH_REPLICAS_AFTER_APPEND) +} else { + (false, Errors.NONE) +} +``` + +这里的判断重点是: + +```plain +leaderLog.highWatermark >= requiredOffset +``` + +requiredOffset是本次写入消息的下一个 offset。如果 high watermark 已经推进到这个 offset,说明这条消息已经被足够的 ISR 副本复制到了,可以认为消息足够安全。 + +这段源码有几个需要注意的点: + ++ `partitionState.isr.size`:当前 ISR 副本数量。 ++ `min.insync.replicas`:topic 或 broker 配置的最小同步副本数。 ++ `requiredAcks == -1`:也就是 producer 的 `acks=all`。 ++ `requiredOffset`:这次 produce 请求需要等待复制完成的目标 offset。 ++ `highWatermark`:Kafka 判断消息是否对消费者可见、是否足够复制的重要边界。 + + + +完整判断可以理解成两段: + +第一段,写入前判断: + +```plain +如果 acks=all,并且 ISR 数量 < min.insync.replicas + -> 拒绝写入 + -> 返回 NOT_ENOUGH_REPLICAS +``` + +第二段,写入后判断: + +```plain +leader 写入成功 + -> 等待 ISR 副本 fetch 复制 + -> follower 追上后推进 high watermark + -> high watermark >= requiredOffset + -> produce 请求返回成功 +``` + +如果消息已经写入 leader,但是等待过程中 ISR 缩小,导致 ISR 数量低于 `min.insync.replicas`,则可能返回: + +```plain +NOT_ENOUGH_REPLICAS_AFTER_APPEND +``` + +也就是:消息已经 append 到 leader 了,但 append 之后 ISR 条件不再满足。 + +Kafka 判断消息是否足够安全,核心看两个条件: + +1. 写入前 ISR 数量是否 >= min.insync.replicas +2. 写入后 high watermark 是否推进到本次 produce 的 requiredOffset + +## 选 leader 时尽量不丢数据 +Kafka 选 leader 时,核心原则是:优先从 ISR 里选 leader。因为 ISR 里的副本表示:这些副本已经和 leader 保持同步,至少已经复制到了 Kafka 认为安全的位置。所以只要新 leader 来自 ISR,就可以最大程度保证不丢已经提交的数据。 + +源码链路: + +```plain +broker 状态变化 / leader 不可用 + -> ReplicationControlManager.generateLeaderAndIsrUpdates() + -> PartitionChangeBuilder + -> electLeader() + -> electAnyLeader() + -> isValidNewLeader() + -> 优先选择 ISR 中可用副本 + -> PartitionChangeRecord.setLeader(...) + -> 写入 metadata log +``` + +关键逻辑在PartitionChangeBuilder。它定义了几种选举类型: + +```plain +public enum Election { + PREFERRED, + ONLINE, + UNCLEAN +} +``` + +这几个类型的含义是: + ++ `PREFERRED`:优先选 preferred replica,但前提是它在 ISR 中并且可用。 ++ `ONLINE`:从 ISR 中选一个可用副本作为 leader。 ++ `UNCLEAN`:优先从 ISR 中选;如果 ISR 里没有可用副本,才允许从 ISR 外选。 + +默认是:Election.ONLINE。也就是说,默认 leader election 是 clean election,只会从 ISR 里选 leader。 + +真正选 leader 的入口是: + +```plain +ElectionResult electLeader() { + if (election == Election.PREFERRED) { + return electPreferredLeader(); + } + + return electAnyLeader(); +} +``` + +普通场景走的是 electAnyLeader(): + +```plain +private ElectionResult electAnyLeader() { + if (isValidNewLeader(partition.leader)) { + return new ElectionResult(partition.leader, false); + } + + Optional onlineLeader = targetReplicas.stream() + .filter(this::isValidNewLeader) + .findFirst(); + + if (onlineLeader.isPresent()) { + return new ElectionResult(onlineLeader.get(), false); + } + + if (election == Election.UNCLEAN) { + Optional uncleanLeader = targetReplicas.stream() + .filter(isAcceptableLeader::test) + .findFirst(); + + if (uncleanLeader.isPresent()) { + return new ElectionResult(uncleanLeader.get(), true); + } + } + + return new ElectionResult(NO_LEADER, false); +} +``` + +这段源码有几个需要注意的点: + ++ 先判断当前 leader 是否仍然有效。 ++ 如果当前 leader 不可用,再从targetReplicas中找一个isValidNewLeader()的副本。 ++ isValidNewLeader()的核心约束就是:副本必须有资格成为 leader,正常情况下要在 ISR 里。 ++ 只有election == Election.UNCLEAN时,才会从 ISR 外选择 leader。 + +也就是说,Kafka 默认不会随便找一个 alive replica 当 leader,而是优先找 ISR 里的副本。 + +这就是“不丢数据”的核心。因为 ISR 里的副本满足: + +```plain +它已经跟上 leader,被 controller 认为是 in-sync replica。 +``` + +所以从 ISR 里选 leader,意味着新 leader 拥有已经提交的数据。如果 ISR 里没有可用副本,会发生什么?默认情况下: + +```plain +没有可用 ISR 副本 + -> 不选 leader + -> partition 进入 unavailable + -> 宁可不可用,也不从 ISR 外选 leader +``` + +这就是 Kafka 默认偏向一致性的地方。只有开启: + +```plain +unclean.leader.election.enable = true +``` + +才允许 unclean leader election。 + +源码里是在 ReplicationControlManager.generateLeaderAndIsrUpdates() 中判断: + +```plain +if (configurationControl.uncleanLeaderElectionEnabledForTopic(topic.name)) { + builder.setElection(PartitionChangeBuilder.Election.UNCLEAN); +} +``` + +配置判断在: + +```plain +configurationControl.uncleanLeaderElectionEnabledForTopic(topic.name) +``` + +如果开启了 unclean election,electAnyLeader()才会走这段: + +```plain +Optional uncleanLeader = targetReplicas.stream() + .filter(isAcceptableLeader::test) + .findFirst(); +``` + +这时选出来的 leader 不一定在 ISR 里,所以可能缺少旧 leader 上已经写入的数据。 + +Kafka 对这种情况会显式标记: + +```plain +if (electionResult.unclean) { + record.setIsr(List.of(electionResult.node)); + record.setLeaderRecoveryState(LeaderRecoveryState.RECOVERING.value()); +} +``` + +这里的含义是: + ++ 如果发生 unclean election,新 leader 会被强制设置成唯一 ISR。 ++ 原来的 ISR 会被替换掉。 ++ leader recovery state 会变成 `RECOVERING`。 ++ 这类选举可能导致数据丢失。 + +源码注释里也写得很直接: + +```plain +// This can result in data loss! +``` + +所以 Kafka 的策略可以总结成两层: + +第一层,默认安全策略: + +```plain +只从 ISR 中选 leader + -> 尽量保证新 leader 拥有已提交数据 + -> 避免数据丢失 +``` + +第二层,可用性兜底策略: + +```plain +如果开启 unclean.leader.election.enable + -> ISR 中无可用副本时,可以从 ISR 外选 leader + -> partition 可以恢复可用 + -> 但可能丢数据 +``` + +总结:Kafka 选 leader 时尽量不丢数据,是因为默认只允许 ISR 中的副本成为 leader;只有显式开启 unclean leader election,才会牺牲数据安全换取可用性。 + diff --git "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" index 7d5a499..3386355 100644 --- "a/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka broker\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -1,4 +1,5 @@ -当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 转载请标明出处 # kafka服务端核心逻辑 目前最新版本kafka已经抛弃了Zookeeper,而是自己维护元数据日志,存放在内部 topic: __cluster_metadata。 diff --git "a/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" index 67f03b8..cecee87 100644 --- "a/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" +++ "b/note/kafka/kafka broker\346\272\220\347\240\201\345\210\206\346\236\220\342\200\224\342\200\224\347\224\237\344\272\247\350\200\205\347\257\207.md" @@ -1,4 +1,5 @@ -当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 转载请标明出处 # KafkaRequestHandler核心流程 `KafkaRequestHandler.run()` 是在 **创建 **`**KafkaRequestHandlerPool**`** 时触发的**。 diff --git "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" index 0faf660..409bc93 100644 --- "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" @@ -1,4 +1,5 @@ -当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 转载请标明出处 # Kafka Rebalance 核心流程 `KafkaConsumer.poll()` 是消费者触发 rebalance 的主要入口。 diff --git "a/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" "b/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" index cefc08a..56143c3 100644 --- "a/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" +++ "b/note/kafka/kafka\346\266\210\350\264\271\350\200\205\346\240\270\345\277\203\346\272\220\347\240\201\345\210\206\346\236\220\357\274\210\344\270\200\357\274\211.md" @@ -1,4 +1,5 @@ -当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) ++ 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:https://github.com/coderbruis/kafka source_code_analysis分支,底层原理持续更新) ++ 转载请标明出处 # kafka消费者核心入口poll kafka消费者核心入口在KafkaConsumer.poll(): From 476082aa40b1f19940becd50f42ef8467dba5bfe Mon Sep 17 00:00:00 2001 From: coderbruis Date: Thu, 2 Jul 2026 11:16:16 +0800 Subject: [PATCH 11/37] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 448b518..29a46b6 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,8 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) - - [Kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) + - [kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) + - [kafka ISR原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) - JDK源码学习 - JDK版本:1.8.0_77 From 27bd0a651fd506302e6e2c3faea8ae2d077e37b9 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Fri, 3 Jul 2026 23:31:13 +0800 Subject: [PATCH 12/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=BD=AC=E8=BD=BD?= =?UTF-8?q?=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...75\215\350\277\220\347\256\227\345\210\206\346\236\220.md" | 2 +- ...267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" | 4 ++-- ...1\240Java volatile\345\205\263\351\224\256\345\255\227.md" | 2 ++ ...72\225\345\261\202\357\274\210\344\270\200\357\274\211.md" | 2 ++ ...72\225\345\261\202\357\274\210\344\272\214\357\274\211.md" | 2 ++ ...Thread\345\272\225\345\261\202\346\272\220\347\240\201.md" | 2 ++ ...ecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" | 3 +++ ...72\220\347\240\201\344\270\216\345\216\237\347\220\206.md" | 2 ++ 8 files changed, 16 insertions(+), 3 deletions(-) diff --git "a/note/JDK/\345\274\200\346\272\220\351\241\271\347\233\256\351\207\214\351\202\243\344\272\233\347\234\213\344\270\215\346\207\202\347\232\204\344\275\215\350\277\220\347\256\227\345\210\206\346\236\220.md" "b/note/JDK/\345\274\200\346\272\220\351\241\271\347\233\256\351\207\214\351\202\243\344\272\233\347\234\213\344\270\215\346\207\202\347\232\204\344\275\215\350\277\220\347\256\227\345\210\206\346\236\220.md" index cec5185..bcbf586 100644 --- "a/note/JDK/\345\274\200\346\272\220\351\241\271\347\233\256\351\207\214\351\202\243\344\272\233\347\234\213\344\270\215\346\207\202\347\232\204\344\275\215\350\277\220\347\256\227\345\210\206\346\236\220.md" +++ "b/note/JDK/\345\274\200\346\272\220\351\241\271\347\233\256\351\207\214\351\202\243\344\272\233\347\234\213\344\270\215\346\207\202\347\232\204\344\275\215\350\277\220\347\256\227\345\210\206\346\236\220.md" @@ -1,4 +1,4 @@ -相信看过几个流行框架源码的小伙伴,或多或少都见到过底层代码运用的位运算,不知道有多少是能够一眼看懂了的,一眼看懂了的都是“真大佬”。如果看不懂的话就老老实实的通过二进制分析来看下这些二进制算法的作用。 ++ 转载请标明出处 ## 1. JDK1.8 HashMap里运用到的为运算 diff --git "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" index 59756a7..ff6c108 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" @@ -1,8 +1,8 @@ ++ 转载请标明出处 + # 是什么?解决什么问题? ThreadLocal是什么?ThreadLocal 是Java提供的一种“线程本地变量”机制。它解决的是:让同一个变量在不同线程中有各自独立的副本。不同线程访问**同一个ThreadLocal对象**时,拿到的值互不影响。 - - # ThreadLocal适用场景 ## 用户上下文场景 这种场景在Web后端很常见:用户登录后,客户端每次请求会带上 token / session / cookie,服务端在过滤器、拦截器或网关层解析出用户信息,然后放到 ThreadLocal 里,后续业务代码就可以直接取当前用户。 diff --git "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Java volatile\345\205\263\351\224\256\345\255\227.md" "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Java volatile\345\205\263\351\224\256\345\255\227.md" index 015b838..2acbf96 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Java volatile\345\205\263\351\224\256\345\255\227.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Java volatile\345\205\263\351\224\256\345\255\227.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + ### 前言 在学习volatile之前,先需要了解并发编程的一些基础概念。 并发编程的目的是为了让程序运行得更快,但是,并不是启动的线程越多就能让程序大幅度的并发执行。因为在实际开发中,并发编程将会面临大量的问题,比如上下文切换问题、死锁问题,以及受限于硬件和软件资源限制问题。 diff --git "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\270\200\357\274\211.md" "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\270\200\357\274\211.md" index d2da4e2..84ff097 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\270\200\357\274\211.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\270\200\357\274\211.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + - [前言](#前言) - [正文](#正文) diff --git "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\272\214\357\274\211.md" "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\272\214\357\274\211.md" index 09008df..944bc32 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\272\214\357\274\211.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240String\346\272\220\347\240\201\344\270\216\345\272\225\345\261\202\357\274\210\344\272\214\357\274\211.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + - [前言](#前言) [正文](#正文) diff --git "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Thread\345\272\225\345\261\202\346\272\220\347\240\201.md" "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Thread\345\272\225\345\261\202\346\272\220\347\240\201.md" index fa16533..1b0dfa6 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Thread\345\272\225\345\261\202\346\272\220\347\240\201.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\255\246\344\271\240Thread\345\272\225\345\261\202\346\272\220\347\240\201.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + ## 前言 在Java中,线程是一个非常重要的知识,大多数开发者对于线程API,属于不用就忘,到用时需要百度的情况,又或者是对线程底层 diff --git "a/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" "b/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" index bb7d39c..ba9b143 100644 --- "a/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" +++ "b/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" @@ -1,3 +1,6 @@ ++ 转载请标明出处 + + # 核心数据结构 ## ctl ctl的作用:在ThreadPoolExecutor中,最核心的一个变量。ctl最核心的作用是设置线程池状态(rs)以及活跃线程数(wc)。 diff --git "a/note/JDK/\346\267\261\345\205\245\350\247\243\350\257\273CompletableFuture\346\272\220\347\240\201\344\270\216\345\216\237\347\220\206.md" "b/note/JDK/\346\267\261\345\205\245\350\247\243\350\257\273CompletableFuture\346\272\220\347\240\201\344\270\216\345\216\237\347\220\206.md" index d3271c8..fc8471d 100644 --- "a/note/JDK/\346\267\261\345\205\245\350\247\243\350\257\273CompletableFuture\346\272\220\347\240\201\344\270\216\345\216\237\347\220\206.md" +++ "b/note/JDK/\346\267\261\345\205\245\350\247\243\350\257\273CompletableFuture\346\272\220\347\240\201\344\270\216\345\216\237\347\220\206.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + - [1 前言](#1-前言) - [2 正文](#2-正文) From 224a67178f236464f75fbe6508c93eca0565d125 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sat, 4 Jul 2026 17:50:14 +0800 Subject: [PATCH 13/37] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=96=87=E7=AB=A0'Nett?= =?UTF-8?q?y=E6=A6=82=E5=BF=B5=E6=89=AB=E7=9B=B2'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 15 +- ...02\345\277\265\346\211\253\347\233\262.md" | 242 ++++++++++++++++++ 2 files changed, 243 insertions(+), 14 deletions(-) create mode 100644 "note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" diff --git a/README.md b/README.md index 29a46b6..f5e4a31 100644 --- a/README.md +++ b/README.md @@ -113,22 +113,9 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - [深入浅出SpringSecurity和OAuth2(三)—— WebSecurity建造核心逻辑](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%B8%89%EF%BC%89%E2%80%94%E2%80%94%20WebSecurity%E5%BB%BA%E9%80%A0%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91.md) - [深入浅出SpringSecurity和OAuth2(四)—— FilterChainProxy过滤器链中的几个重要的过滤器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E5%9B%9B%EF%BC%89%E2%80%94%E2%80%94%20FilterChainProxy%E8%BF%87%E6%BB%A4%E5%99%A8%E9%93%BE%E4%B8%AD%E7%9A%84%E5%87%A0%E4%B8%AA%E9%87%8D%E8%A6%81%E7%9A%84%E8%BF%87%E6%BB%A4%E5%99%A8.md) -- Dubbo底层源码解析 - - Dubbo底层源码版本:2.7.8 - - [Dubbo底层源码学习—— 源码搭建](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo%E6%BA%90%E7%A0%81%E6%90%AD%E5%BB%BA.md) - - [Dubbo底层源码学习(一)—— Dubbo的URL](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%20Dubbo%E7%9A%84URL.md) - - [Dubbo底层源码学习(二)—— Dubbo的SPI机制(上)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20Dubbo%E7%9A%84SPI%E6%9C%BA%E5%88%B6%EF%BC%88%E4%B8%8A%EF%BC%89.md) - - [Dubbo底层源码学习(二)—— Dubbo的SPI机制(中)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo底层源码学习%EF%BC%88二%EF%BC%89——%20Dubbo的SPI机制%EF%BC%88中%EF%BC%89.md -) - - [Dubbo底层源码学习(二)—— Dubbo的SPI机制(下)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20Dubbo%E7%9A%84SPI%E6%9C%BA%E5%88%B6%EF%BC%88%E4%B8%8B%EF%BC%89.md) - - Dubbo底层源码学习(三)—— Dubbo的注册中心 - - Dubbo底层源码学习(四)—— Dubbo的注册中心缓存机制 - - Dubbo底层源码学习(五)—— Dubbo的注册中心重试机制 - - [Dubbo底层源码学习(六)—— Dubbo的服务暴露](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Dubbo/Dubbo%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E2%80%94%E2%80%94%E6%9C%8D%E5%8A%A1%E6%9A%B4%E9%9C%B2.md) - - Dubbo底层源码学习(七)—— Dubbo的服务消费 - - Netty底层源码解析 - Netty版本:4.1.43.Final + - [Netty概念扫盲]() - [二进制运算以及源码、反码以及补码学习](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/%E4%BA%8C%E8%BF%9B%E5%88%B6.md) - [Netty源码包结构](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E6%BA%90%E7%A0%81%E5%8C%85%E7%BB%93%E6%9E%84.md) - [Netty底层源码解析-EventLoopGroup](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E4%B8%AD%E7%9A%84EventLoopGroup%E6%98%AF%E4%BB%80%E4%B9%88.md) diff --git "a/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" "b/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" new file mode 100644 index 0000000..b41d47d --- /dev/null +++ "b/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" @@ -0,0 +1,242 @@ +# EventLoop +EventLoop 是Netty的事件循环线程模型,它负责: + + - 监听 I/O 事件,比如连接、读、写 + - 执行Channel相关任务 + - 驱动 pipeline 里的 handler 回调 + - 保证同一个Channel的事件通常在同一个线程串行执行 + + + +EventLoop解决以下问题: + + - 高效管理大量连接 + - 避免一个连接一个线程的高成本 + - 减少并发锁竞争 + - 让网络事件处理更有序、可控 + + + +一句话:EventLoop 就是 Netty 用来轮询 I/O 事件并执行 Channel 任务的核心线程。 + +## EventLoop实现 +EventLoop是通用抽象,负责: + + - 执行任务 + - 注册/管理Channel + - 处理 I/O 事件 + + + +具体底层怎么等 I/O 事件,由传输实现决定: + + - NIO:基于 Java Selector + - epoll:基于 Linux epoll + - kqueue:基于 BSD/macOS kqueue + - io_uring:基于 Linux io_uring + + + +EventLoop 通用调度模型,Transport / IoHandler 决定底层 I/O 轮询实现。 + + + +## 网络事件\socket是什么 +网络事件就是 socket 状态变化或 I/O 就绪通知。常见包括: + + - 有新连接进来:accept + - 连接建立完成:connect + - 有数据可读:read + - 可以继续写数据:write + - 连接关闭或异常:close / error + +操作系统告诉 Netty“这个连接现在可以做某件事了”,这就是网络事件。 + + + +那socket是什么呢? + +Socket是应用程序和网络之间的通信端点。可以理解成:程序用来收发网络数据的“连接句柄”。比如 TCP 通信里: + +```plain +客户端 Socket <====网络====> 服务端 Socket +``` + +程序通过 socket: + + - 连接远程服务 + - 发送数据 + - 接收数据 + - 关闭连接 + + + +socket 是程序进行网络通信的入口。 + +# Pipeline +Pipeline 是 Netty 里处理请求/响应的流水线,里面的 Handler 分步骤完成拆包、编解码、序列化、业务分发。Netty Client和Netty Server的pipeline都是用的DefaultChannelPipeline,但是怎么区分是Netty Client的pipeline还是Netty Server的pipeline呢? + +靠 Channel 类型和创建来源 区分,不靠 DefaultChannelPipeline 类型区分。DefaultChannelPipeline 本身是通用实现,Client 和 Server 都用它。但它绑定的 Channel 不同: + +```plain +客户端连接: +NioSocketChannel + -> new DefaultChannelPipeline(this) + -> Client Pipeline + +服务端监听端口: +NioServerSocketChannel + -> new DefaultChannelPipeline(this) + -> Server Boss Pipeline + +服务端接收到的客户端连接: +NioSocketChannel + -> new DefaultChannelPipeline(this) + -> Server Child Pipeline +``` + +也就是说: + +```plain +pipeline.channel() +``` + + Pipeline 类型都一样,属于谁取决于它绑定的是哪个 Channel,以及这个 Channel 是由客户端 Bootstrap创建,还是服务端ServerBootstrap创建。 + + + +一个pipeline既可以处理入站请求,也可以处理出站请求,只不过方向不一样。 + +```plain +入站 Inbound:读数据、解码、业务处理 +出站 Outbound:写数据、编码、发送 + +入站:Head -> Tail +出站:Tail -> Head +``` + + + +```plain +一个 ChannelPipeline +┌──────────────────────────────────────────────┐ +│ │ +│ Head ── H1 ── H2 ── H3 ── H4 ── Tail │ +│ │ +└──────────────────────────────────────────────┘ + + +入站 Inbound:从 Head 到 Tail +Socket 收到数据 + ↓ +Head ──> H1 ──> H2 ──> H3 ──> H4 ──> Tail + 解码 反序列化 业务处理 + + +出站 Outbound:从 Tail 到 Head +业务调用 writeAndFlush() + ↓ +Tail ──> H4 ──> H3 ──> H2 ──> H1 ──> Head + 序列化 编码 写 Socket +``` + +## 出站和入站 +Netty出站——ChannelInboundHandler,是指当前 Channel 将数据写出到网络。比如 RPC Client 发送请求,或 RPC Server 返回响应,都会经过出站流程:Java 对象序列化、协议编码/封装,转成 ByteBuf,再通过 Socket 发送出去。 + +Netty入站——ChannelOutboundHandler,是指当前 Channel 从网络读取数据。比如 RPC Server 接收请求,或 RPC Client 接收响应,都会经过入站流程:读取 ByteBuf,按协议处理粘包半包,解码协议,再反序列化成 Java 对象。 + +## 为什么入站:HEAD->TAIL,出站:TAIL->HEAD? +核心逻辑就是:**Head 更靠近 Socket / 底层 I/O,Tail 更靠近业务处理端。** + +因为 Netty 把 `Head` 设计成**靠近底层 Socket 的入口**,`Tail` 设计成**靠近业务处理的末端**。 + +所以网络数据进来时: + +```plain +Socket 读到数据 + ↓ +HeadContext + ↓ +InboundHandler1 + ↓ +InboundHandler2 + ↓ +TailContext +``` + +也就是: + +```plain +Head -> Tail +``` + +反过来,业务要写数据出去时,是从业务侧往 Socket 走: + +```plain +Tail -> Head +``` + +最后到 `HeadContext`,由它调用底层 unsafe 把数据写到 Socket。 + +# Channel +Channel是Netty对网络连接/通信端点的抽象,也就是Netty里对 socket 连接或监听端口的封装。它代表一个可进行 I/O 操作的对象,比如: + ++ 服务端监听端口:ServerSocketChannel ++ 客户端连接:SocketChannel ++ UDP 通信:DatagramChannel + + + +它主要负责: + ++ 读写数据 ++ 绑定/连接/关闭 ++ 持有关联的EventLoop ++ 持有ChannelPipeline + +# EventLoopGroup、EventLoop、Channel关系 +在Netty中,一般这样设置: + +```plain +EventLoopGroup bossGroup = new NioEventLoopGroup(1); +EventLoopGroup workerGroup = new NioEventLoopGroup(); +``` + +bossGroup:负责监听端口、accept 新连接,通常 1 个线程就够。workerGroup:负责已连接客户端的读写 I/O,默认线程数通常是 CPU核数 * 2。 + + + +EventLoop、线程、channel之间的关系: + + - 1 个 EventLoop 通常绑定 1 个线程 + - 1 个 EventLoopGroup 包含多个 EventLoop + - 1 个 Channel 注册到 1 个 EventLoop + - 1个EventLoop会有多个channel,一个EventLoop会轮训多个channel的事件 + + + +在Netty中,一个EventLoop线程不是被某个Channel独占的。它会轮询处理多个Channel的事件: + +```plain +EventLoop-1 + -> Channel A read + -> Channel B read + -> Channel C write + -> 定时任务 + -> 下一轮 select +``` + + + +总结:BossGroup 里通常配置一个或少数几个 EventLoop。Netty 服务端调用 bind() 时,会创建并将 XxxServerSocketChannel 注册到 BossGroup 的某个 EventLoop 上。这个 EventLoop 主要负责轮询 XxxServerSocketChannel 上的客户端新连接事件,也就是 TCP accept 事件。当 accept 到新连接后,Netty 会创建对应的 XxxSocketChannel,并将它注册到 WorkerGroup 按策略选出的某个 EventLoop 上。后续该客户端连接的 read/write 事件,都由这个 XxxSocketChannel 绑定的 Worker EventLoop 负责轮询和处理。 + +不同的EventLoop底层使用的轮询机制不一样,不管底层是用的:Selector / epoll / kqueue / io_uring / 阻塞 I/O,都需要轮询事件。 + + - NioEventLoop -> Selector + - EpollEventLoop -> Linux epoll + - KQueueEventLoop -> BSD/macOS kqueue + - IoUringEventLoop -> Linux io_uring + - OioEventLoop -> 阻塞 I/O 模型 + + + From d11d5e4404a9ec843a81a93a5840f3d3068ffec1 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sat, 4 Jul 2026 17:51:34 +0800 Subject: [PATCH 14/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5e4a31..ddc0406 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - Netty底层源码解析 - Netty版本:4.1.43.Final - - [Netty概念扫盲]() + - [Netty概念扫盲](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - [二进制运算以及源码、反码以及补码学习](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/%E4%BA%8C%E8%BF%9B%E5%88%B6.md) - [Netty源码包结构](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E6%BA%90%E7%A0%81%E5%8C%85%E7%BB%93%E6%9E%84.md) - [Netty底层源码解析-EventLoopGroup](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E4%B8%AD%E7%9A%84EventLoopGroup%E6%98%AF%E4%BB%80%E4%B9%88.md) From 5bd76b88a15f4033aab9f573339603a5b34c0f45 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sun, 5 Jul 2026 20:54:55 +0800 Subject: [PATCH 15/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0kafka=E6=A0=B8=E5=BF=83?= =?UTF-8?q?=E6=A6=82=E5=BF=B5=E6=89=AB=E7=9B=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- ...02\345\277\265\346\211\253\347\233\262.md" | 2 + ...02\345\277\265\346\211\253\347\233\262.md" | 1129 +++++++++++++++++ 3 files changed, 1132 insertions(+), 1 deletion(-) create mode 100644 "note/kafka/Kafka\346\240\270\345\277\203\346\246\202\345\277\265\346\211\253\347\233\262.md" diff --git a/README.md b/README.md index ddc0406..abff78b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - kafka源码分析 - kafka版本:4.2 + - [kafka 核心概念扫描]() - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) @@ -129,7 +130,6 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - Netty底层源码解析-FastThreadLocal原理分析 - Netty底层源码解析-内存分配原理分析 - Netty底层源码解析-RocketMQ底层使用到的Netty - - [实战+原理效果更佳!强烈推荐闪电侠大佬实战课:《Netty 入门与实战:仿写微信 IM 即时通讯系统》](https://juejin.cn/book/6844733738119593991) Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下,有需要的读者可前往查看。 diff --git "a/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" "b/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" index b41d47d..9eb1d45 100644 --- "a/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" +++ "b/note/Netty/Netty\346\246\202\345\277\265\346\211\253\347\233\262.md" @@ -1,3 +1,5 @@ ++ 转载请标明出处 + # EventLoop EventLoop 是Netty的事件循环线程模型,它负责: diff --git "a/note/kafka/Kafka\346\240\270\345\277\203\346\246\202\345\277\265\346\211\253\347\233\262.md" "b/note/kafka/Kafka\346\240\270\345\277\203\346\246\202\345\277\265\346\211\253\347\233\262.md" new file mode 100644 index 0000000..ab97ca5 --- /dev/null +++ "b/note/kafka/Kafka\346\240\270\345\277\203\346\246\202\345\277\265\346\211\253\347\233\262.md" @@ -0,0 +1,1129 @@ ++ 转载请标明出处 + + + +本章旨在快速扫盲\回顾kafka关键概念以及定义,不做原理深入分析。 + +# Producer +Kafka Producer 是 **消息生产者**,负责把业务数据写入 Kafka Topic。 + +它主要解决: + +1. **发送消息** +把应用产生的事件、日志、订单、状态变更写入 Kafka。 +2. **选择分区** +决定消息写到 Topic 的哪个 Partition。 +3. **提高吞吐** +通过批量发送、压缩、异步发送提升写入性能。 +4. **保证可靠性** +通过 `acks`、重试、幂等、事务控制写入语义。 + +**Producer 的原理** + +Producer 发送消息大致流程: + +```plain +业务线程 + -> 序列化 key/value + -> 选择 Partition + -> 写入本地缓冲区 + -> Sender 线程批量发送给 Partition Leader + -> Broker 返回 ack +``` + +分区选择规则: + +```plain +指定 partition:直接写入该 partition +有 key:通常按 key hash 选择 partition +无 key:默认策略尽量均衡分散写入 +``` + +Producer 不直接写所有副本,只写 **Partition Leader**: + +```plain +Producer -> Leader Replica -> Follower Replica 同步 +``` + +写入是否成功,取决于 `acks`: + +```plain +acks=0 不等响应,最快但可能丢 +acks=1 Leader 写入成功即返回 +acks=all 等 ISR 中足够副本确认后返回,最可靠 +``` + +**关键机制** + +`batch.size`:控制批次大小,提高吞吐。 + +`linger.ms`:等待凑批时间,增大可提升吞吐但增加延迟。 + +`compression.type`:压缩消息,减少网络和磁盘开销。 + +`retries`:失败自动重试。 + +`enable.idempotence=true`:启用幂等,避免重试导致重复写入。 + +`transactional.id`:启用事务,支持跨分区原子写入和 Exactly Once。 + +**需要注意** + ++ Producer 是异步批量发送,不是每条消息都立刻发到 Broker。 ++ 相同 key 通常进入同一 Partition,可保证 key 级顺序。 ++ Kafka 只保证单 Partition 内有序,不保证 Topic 全局有序。 ++ 可靠写入常用: + +```plain +acks=all +enable.idempotence=true +retries=2147483647 +``` + ++ 更强可靠性还要配合 Topic: + +```plain +replication.factor=3 +min.insync.replicas=2 +unclean.leader.election.enable=false +``` + +一句话总结: + +**Producer 是 Kafka 的写入端,负责序列化消息、选择分区、批量压缩发送到 Partition Leader,并通过 ack、重试、幂等和事务控制吞吐与可靠性的平衡。** + +# Consumer +Kafka Consumer 是 **消息消费者**,负责从 Kafka Topic 中拉取消息并交给业务处理。 + +它主要解决: + +1. **读取消息** +从 Topic 的 Partition 中按 offset 拉取数据。 +2. **记录进度** +通过提交 offset 记录自己消费到哪里。 +3. **水平扩展** +多个 Consumer 组成 Consumer Group,共同消费多个 Partition。 +4. **故障转移** +某个 Consumer 挂掉后,它负责的 Partition 会通过 Rebalance 分给其他 Consumer。 + +**Consumer 的原理** + +Consumer 是 **主动拉取模型**,不是 Broker 推送。 + +基本流程: + +```plain +Consumer 订阅 Topic + -> 加入 Consumer Group + -> Coordinator 分配 Partition + -> Consumer 从 Partition Leader 拉取消息 + -> 业务处理消息 + -> 提交 offset +``` + +Consumer 实际消费的是 Partition: + +```plain +Topic + ├── Partition-0 -> Consumer-A + ├── Partition-1 -> Consumer-B + └── Partition-2 -> Consumer-A +``` + +同一个 Consumer Group 内: + +```plain +一个 Partition 同一时刻只能被一个 Consumer 消费 +一个 Consumer 可以消费多个 Partition +``` + +不同 Consumer Group 之间互不影响,可以各自完整消费同一份数据。 + +**Offset 机制** + +Offset 是 Consumer 的消费进度: + +```plain +Partition-0: offset 0, 1, 2, 3... +``` + +Consumer 处理完消息后提交 offset。提交的位置通常表示: + +```plain +下一条要消费的 offset +``` + +Offset 默认存储在 Kafka 内部 topic: + +```plain +__consumer_offsets +``` + +提交方式: + +```plain +自动提交:简单,但可能重复或丢处理语义 +手动提交:可控,生产更常用 +``` + +**需要注意** + ++ Kafka 保证 Partition 内有序,不保证 Topic 全局有序。 ++ Consumer 数量超过 Partition 数,多出来的 Consumer 会空闲。 ++ `max.poll.interval.ms` 控制两次 poll 的最大间隔,处理太慢可能被踢出 Group。 ++ `session.timeout.ms` 和 `heartbeat.interval.ms` 控制心跳存活。 ++ Rebalance 会导致短暂停顿,也可能带来重复消费。 ++ 业务要按“至少一次”语义设计,处理逻辑最好支持幂等。 + +**常见关键配置** + +```plain +group.id=order-consumer +enable.auto.commit=false +auto.offset.reset=latest +max.poll.records=500 +max.poll.interval.ms=300000 +partition.assignment.strategy=org.apache.kafka.clients.consumer.CooperativeStickyAssignor +``` + +事务场景: + +```plain +isolation.level=read_committed +``` + +一句话总结: + +**Consumer 是 Kafka 的读取端,通过主动拉取 Partition 数据、提交 offset 记录进度,并依靠 Consumer Group 和 Rebalance 实现并行消费与故障转移。** + +# Broker +Kafka Broker 是 **Kafka 集群中的服务节点**。 + +它主要负责: + +1. **存储数据** +保存 Topic 的 Partition 日志文件。 +2. **处理读写请求** +Producer 写入消息,Consumer 拉取消息,最终都由对应 Partition 的 Leader Broker 处理。 +3. **副本同步** +Leader Broker 接收写入,Follower Broker 从 Leader 拉取数据复制。 +4. **参与集群管理** +Broker 上下线会影响 Leader 选举、ISR 变化、分区迁移等。 + +**Broker 的原理** + +一个 Kafka 集群由多个 Broker 组成: + +```plain +Kafka Cluster + ├── Broker-1 + ├── Broker-2 + └── Broker-3 +``` + +Topic 被拆成多个 Partition,Partition 分布在不同 Broker 上: + +```plain +Topic order + ├── Partition-0 Leader -> Broker-1 + ├── Partition-1 Leader -> Broker-2 + └── Partition-2 Leader -> Broker-3 +``` + +客户端不会固定读写某一台 Broker,而是先获取元数据,再找到目标 Partition 的 Leader: + +```plain +Client -> 获取 Metadata +Client -> Partition Leader 所在 Broker +``` + +Broker 本地会把 Partition 存成追加日志: + +```plain +log segment + index + timeindex +``` + +如果一个 Broker 宕机,它上面的 Leader Partition 会触发 Leader 选举,由其他 ISR 副本接管。 + +**需要注意** + ++ Broker 是物理/进程节点,Topic/Partition 是逻辑数据结构。 ++ 一个 Broker 可以存多个 Topic 的多个 Partition。 ++ 一个 Partition 的 Leader 只能在一个 Broker 上。 ++ Broker 越多,集群容量和并行能力越强,但运维复杂度也更高。 ++ 客户端连接是否正常,常和 `listeners`、`advertised.listeners` 配置有关。 ++ Broker 磁盘、网络、页缓存对 Kafka 性能影响很大。 + +一句话总结: + +**Broker 是 Kafka 的服务节点,负责承载 Partition 数据、处理客户端读写、副本复制和故障切换,是 Kafka 集群容量、性能和可用性的基础。** + +# Offset +Kafka Offset 是 **消息在 Partition 内的位置编号**。 + +它主要用于: + +1. **定位消息** +每条消息在某个 Partition 内都有唯一 offset。 +2. **记录消费进度** +Consumer 通过提交 offset 表示自己消费到哪里。 +3. **支持断点续消** +Consumer 重启后,可以从上次提交的 offset 继续消费。 +4. **支持回溯消费** +可以手动 seek 到某个 offset,重新消费历史消息。 + +**Offset 的原理** + +Kafka 的消息存储在 Partition 中,Partition 是有序追加日志: + +```plain +Partition-0: +offset 0 -> offset 1 -> offset 2 -> offset 3 +``` + +Offset 只在 **单个 Partition 内有意义**: + +```plain +Topic 全局没有统一 offset +不同 Partition 的 offset 互不相关 +``` + +Consumer 拉取消息后,会提交自己的消费进度。提交的 offset 通常表示: + +```plain +下一条要消费的 offset +``` + +例如处理完 `offset=100`,通常提交: + +```plain +offset=101 +``` + +Consumer Group 的 offset 默认存储在 Kafka 内部 Topic: + +```plain +__consumer_offsets +``` + +**需要注意** + ++ Offset 不是消息 ID,只是 Partition 内的位置。 ++ Kafka 保证 Partition 内 offset 递增有序。 ++ Consumer 提交 offset 不代表业务一定成功,取决于提交时机。 ++ 自动提交简单,但可能造成重复消费或处理语义不清。 ++ 生产中常用手动提交:业务处理成功后再提交 offset。 ++ 如果 offset 过期或不存在,会按 `auto.offset.reset` 处理: + +```plain +earliest +latest +none +``` + +**几个相关概念** + +```plain +LEO:Log End Offset,日志末尾下一条 offset +HW :High Watermark,消费者可见的已提交边界 +Committed Offset:Consumer Group 已提交的消费进度 +``` + +一句话总结: + +**Offset 是 Kafka 在 Partition 内定位消息和记录消费进度的核心机制;Consumer 通过提交 offset 实现断点续消,但是否重复或丢处理,取决于业务处理和 offset 提交的顺序。** + +# ISR +ISR 是 **In-Sync Replicas,同步副本集合**。 + +它表示:**当前和 Leader 保持同步、具备成为新 Leader 资格的副本集合**。 + +主要作用: + +1. **保证数据可靠性** +Producer 使用 `acks=all` 时,Leader 需要等待 ISR 中足够副本确认。 +2. **控制 Leader 选举范围** +Leader 挂掉后,Kafka 通常只从 ISR 中选新 Leader,避免选到落后副本导致数据丢失。 +3. **推进 HW** +Kafka 根据 ISR 中副本的复制进度推进 High Watermark,消费者只能读取 HW 之前的数据。 + +**ISR 的原理** + +每个 Partition 都维护自己的 ISR: + +```plain +Partition-0 +Leader: broker-1 +ISR: [broker-1, broker-2, broker-3] +``` + +Follower 会不断向 Leader 拉取消息。如果某个 Follower 长时间没追上 Leader,就会被移出 ISR。 + +关键配置: + +```plain +replica.lag.time.max.ms +``` + +如果 Follower 在这个时间内没有保持有效同步,就会被踢出 ISR。 + +例如: + +```plain +Leader LEO = 100 +Follower-2 LEO = 100 +Follower-3 LEO = 70 +``` + +如果 Follower-3 落后太久: + +```plain +ISR: [broker-1, broker-2] +``` + +之后 HW 推进和 `acks=all` 确认就不再等待 broker-3。当 broker-3 后续追上 Leader 后,可以重新加入 ISR。 + + + +**和 acks/min.insync.replicas 的关系** + +常见可靠配置: + +```plain +replication.factor=3 +min.insync.replicas=2 +acks=all +``` + +含义是: + +```plain +3 个副本 +至少 2 个 ISR 副本可用 +Producer 写入才算成功 +``` + +如果 ISR 只剩 1 个,`acks=all` 写入会失败,避免单副本写入带来的数据风险。 + +**需要注意** + ++ ISR 不是所有副本,所有副本叫 AR。 ++ ISR 是动态变化的,Follower 落后会被踢出,追上后会加入。 ++ ISR 中的副本不一定和 Leader 完全一样新,但必须满足同步要求。 ++ Leader 通常必须在 ISR 中。 ++ `unclean.leader.election.enable=true` 时,非 ISR 副本也可能当 Leader,但可能导致数据丢失。 + +一句话总结: + +**ISR 是 Kafka 判断副本是否“足够同步”的核心机制,它决定了写入确认、HW 推进和 Leader 故障切换的安全边界。** + +# HW +Kafka 中 **HW** 是 **High Watermark,高水位线**。 + +它表示:**一个分区中已经被所有 ISR 副本确认复制到的位置**。消费者只能读取到 HW 之前的消息,也就是 Kafka 认为“已经提交、不会丢失”的消息。 + +简单说: + +```plain +Log End Offset, LEO: 某个副本本地日志的末尾位置 +High Watermark, HW: ISR 副本都已经复制到的最大安全位置 +``` + +例如一个分区有 3 个副本: + +```plain +Leader LEO = 10 +Follower1 LEO = 10 +Follower2 LEO = 8 +``` + +那么 HW 最多只能推进到 `8`,因为 offset `8` 之后的消息还没有被所有 ISR 副本复制到。 + +**HW 解决的问题** + +主要解决两个问题: + +1. **防止消费者读到可能丢失的消息** + +如果消费者能直接读取 Leader 的最新 LEO,那么它可能读到一条还没复制到 Follower 的消息。此时 Leader 宕机,新 Leader 可能没有这条消息,这条消息就会被截断,消费者之前读到的数据就“消失”了。HW 限制消费者只能读已经被 ISR 副本确认的消息,避免这种不一致。 + +2. **Leader 切换时保证日志一致性** + +Kafka 发生 Leader 选举后,新 Leader 会基于 HW 进行日志截断。超过 HW 的数据如果没有被提交,可能会被删除,以保证各副本日志一致。 + + + +所以 HW 本质上是 Kafka 的 **提交边界**: + +```plain +小于 HW 的消息:已提交,对消费者可见 +大于等于 HW 的消息:未完全确认,对消费者不可见,可能被截断 +``` + +一句话总结: +** HW 是 Kafka 用来标记“哪些消息已经足够安全,可以对消费者可见”的机制,主要用于保证副本一致性和避免消费者读到未来可能丢失的数据。** + +# Partition +Kafka 中的 **Partition 是 Topic 的物理分片**。一个 Topic 可以拆成多个 Partition。 + +它主要解决三个问题: + +1. **提升吞吐** +多个 Partition 可以分布在不同 Broker 上,Producer、Consumer 可以并行读写。 +2. **支持扩展** +Consumer Group 中,一个 Partition 同一时刻只能被同组内一个 Consumer 消费;Partition 越多,消费并行度上限越高。 +3. **保证局部顺序** +Kafka 不保证整个 Topic 全局有序,只保证 **单个 Partition 内消息按 offset 有序**。相同 key 通常会被写入同一个 Partition,从而保证 key 级顺序。 + +**Partition 的原理** + +Producer 写消息时,会根据规则选择 Partition: + +```plain +有 key:通常按 key hash 选择 partition +无 key:按默认分区策略分散写入 +指定 partition:直接写入指定 partition +``` + +每个 Partition 本质是一段有序追加日志: + +```plain +Partition-0: +offset 0 -> offset 1 -> offset 2 -> offset 3 +``` + +每条消息在 Partition 内都有一个递增的 `offset`,Consumer 通过保存 offset 来记录消费进度。 + +为了高可用,每个 Partition 可以有多个副本: + +```plain +Partition-0: +Leader Replica 处理读写 +Follower Replica 从 Leader 同步 +``` + +正常情况下,客户端只和 Leader 副本交互;Follower 负责复制。当 Leader 挂掉时,Kafka 会从 ISR 中选新的 Leader。 + +**需要注意** + ++ Partition 数决定消费并行度上限。 ++ Partition 内有序,Topic 整体不保证全局有序。 ++ Partition 数可以增加,但增加后 key 到 partition 的映射可能变化,影响顺序语义。 ++ Partition 太多会增加 Broker、Controller、文件句柄、内存和故障恢复成本。 ++ 一个 Consumer Group 内,同一个 Partition 不能同时被多个 Consumer 消费。 + +一句话总结: + +**Partition 是 Kafka 实现高吞吐、水平扩展和局部有序的核心机制;它把 Topic 拆成多个有序日志分片,每个分片可独立读写、复制和故障切换。** + +# Replica +Kafka 副本是 **Partition 的多份拷贝**,主要解决: + +1. **高可用** +Leader 副本挂了,可以从其他同步副本中选新 Leader。 +2. **防止数据丢失** +消息不只存在一台 Broker 上,降低单机故障导致数据丢失的风险。 +3. **故障恢复** +Broker 重启后,副本可以继续从 Leader 拉取缺失数据,重新追上进度。 + +**副本的原理** + +每个 Partition 有多个副本: + +```plain +Partition-0: +Leader Replica 处理读写 +Follower Replica 从 Leader 拉取日志 +Follower Replica 从 Leader 拉取日志 +``` + +Kafka 默认客户端读写都走 **Leader 副本**。Follower 不直接处理普通客户端请求,而是像 Consumer 一样向 Leader 发送 fetch 请求,同步日志。 + +Kafka 用 **ISR** 管理同步副本: + +```plain +ISR = In-Sync Replicas,同步副本集合 +``` + +只有跟得上 Leader 的副本才在 ISR 中。Leader 根据 ISR 中副本的复制进度推进 **HW** + +```plain +HW = High Watermark,高水位线 +``` + +消费者只能读 HW 之前的数据,因为这些数据被认为已经提交、相对安全。如果 Leader 宕机,Kafka 通常从 ISR 中选新的 Leader,保证新 Leader 拥有已提交数据。 + +**需要注意** + ++ 副本数常见配置是 `replication.factor=3`。 ++ 副本不是越多越好,副本越多,磁盘和网络复制成本越高。 ++ `acks=all` 配合 `min.insync.replicas` 才能真正提升写入可靠性。 ++ 非 ISR 副本如果被选为 Leader,可能导致数据丢失,所以生产通常关闭: + +```plain +unclean.leader.election.enable=false +``` + ++ Follower 同步是主动拉取,不是 Leader 推送。 + +一句话总结: + +**Kafka 副本机制通过 Leader 处理读写、Follower 复制日志、ISR 判断同步状态,在 Broker 故障时保证分区仍可用,并尽量避免已提交数据丢失。** + +# Rebalance +Kafka 中的重平衡是 **Consumer Group 内部分区归属重新分配** 的过程。 + +它主要解决: + +1. **消费者扩缩容** +新 Consumer 加入后,把部分 Partition 分给它,提高并行消费能力。 +2. **消费者故障转移** +某个 Consumer 下线后,它负责的 Partition 会转移给其他 Consumer。 +3. **Topic/Partition 变化** +Topic 增加分区后,Consumer Group 需要重新分配新 Partition。 + +**重平衡的原理** + +同一个 Consumer Group 里: + +```plain +一个 Partition 同一时刻只能分配给一个 Consumer +一个 Consumer 可以消费多个 Partition +``` + +Group Coordinator 负责管理 Consumer Group。Consumer 通过心跳维持成员身份。 + +触发重平衡的常见情况: + +```plain +Consumer 加入 group +Consumer 离开 group +Consumer 心跳超时 +订阅的 topic 分区数变化 +Consumer 长时间不 poll +``` + +重平衡大致流程: + +```plain +1. Consumer 加入/离开或状态变化 +2. Group Coordinator 触发 rebalance +3. 选出一个 Consumer 作为 Group Leader +4. Group Leader 根据分配策略计算 Partition 分配方案 +5. Coordinator 把分配结果下发给各 Consumer +6. Consumer 从新分配的 Partition 继续消费 +``` + +常见分配策略: + +```plain +RangeAssignor +RoundRobinAssignor +StickyAssignor +CooperativeStickyAssignor +``` + +现代生产环境更推荐 `CooperativeStickyAssignor`,因为它是增量重平衡,可以减少整体停顿。 + +**需要注意** + ++ 重平衡期间,相关 Consumer 可能暂停消费。 ++ 频繁重平衡会导致消费抖动、延迟升高、重复消费增加。 ++ `max.poll.interval.ms` 太小,业务处理慢,会导致 Consumer 被踢出 group。 ++ `session.timeout.ms` 和 `heartbeat.interval.ms` 控制心跳失效判断。 ++ Consumer 数量超过 Partition 数时,多出来的 Consumer 会空闲。 ++ 手动提交 offset 时,要在分区被撤销前处理好提交,避免重复消费或丢处理进度。 + +**常见优化** + +```plain +partition.assignment.strategy=org.apache.kafka.clients.consumer.CooperativeStickyAssignor +max.poll.interval.ms=适当调大 +max.poll.records=适当调小 +session.timeout.ms=适当设置 +heartbeat.interval.ms=session.timeout.ms 的 1/3 左右 +``` + +如果业务处理很慢,可以减少每次拉取数量: + +```plain +max.poll.records=100 +``` + +或者把拉取和处理解耦,避免长时间不调用 `poll()`。 + +一句话总结: + +**重平衡是 Kafka Consumer Group 为了适应消费者数量、分区数量和故障变化而重新分配 Partition 的机制;它保证负载动态转移,但频繁发生会带来消费暂停、延迟上升和重复消费风险。** + +# Topic +Kafka 中的 **Topic 是消息的逻辑分类**,类似消息队列里的“主题”或“类别”。 + +它主要用于: + +1. **业务隔离** +不同业务数据放到不同 Topic,比如: + +```plain +order-events +payment-events +user-log +``` + +2. **发布订阅** +Producer 向 Topic 写消息,Consumer 订阅 Topic 读消息。 +3. **数据管理** +Topic 可以配置保留时间、分区数、副本数、清理策略等。 + +**Topic 的原理** + +Topic 本身只是逻辑概念,真正存储数据的是它下面的 **Partition**。 + +```plain +Topic + ├── Partition-0 + ├── Partition-1 + └── Partition-2 +``` + +Producer 写入 Topic 时,Kafka 会把消息分配到某个 Partition: + +```plain +有 key:通常按 key hash 分区 +无 key:按策略均衡分区 +``` + +Consumer 订阅 Topic 后,本质上是从 Topic 的多个 Partition 中拉取消息。 + +每个 Partition 是一段有序日志: + +```plain +Partition-0: offset 0, 1, 2, 3... +Partition-1: offset 0, 1, 2, 3... +``` + +所以 Kafka 保证的是 **Partition 内有序**,不是 Topic 全局有序。 + +**需要注意** + ++ Topic 是逻辑分类,Partition 才是物理存储和并行单位。 ++ 一个 Topic 可以有多个 Consumer Group 独立消费。 ++ Topic 的分区数影响吞吐和消费并行度。 ++ Topic 的副本数影响高可用和数据可靠性。 ++ Topic 可以配置 `delete` 或 `compact` 清理策略。 ++ 生产环境通常关闭自动创建 Topic,避免误创建: + +```plain +auto.create.topics.enable=false +``` + +一句话总结: + +**Topic 是 Kafka 对消息流的逻辑命名和管理单元;它通过多个 Partition 实现并行读写,通过副本实现高可用,通过消费组实现一份数据被多类业务独立消费。** + +# KRaft +KRaft 是 Kafka 的自管理元数据机制,用来替代 ZooKeeper。 + +它主要解决: + +1. **去掉 ZooKeeper 依赖** +Kafka 不再需要单独维护 ZooKeeper 集群,部署和运维更简单。 +2. **统一元数据管理** +Topic、Partition、Broker、ACL、配置等元数据由 Kafka 自己管理。 +3. **提升扩展性和恢复效率** +元数据变更通过 Kafka 内部的 Raft 日志复制,Controller 可以更快恢复状态。 + +**KRaft 的原理** + +KRaft 基于 **Raft 共识协议**。 + +Kafka 集群中会有一组 **Controller 节点** 组成 quorum: + +```plain +Controller Quorum + ├── Controller 1 + ├── Controller 2 + └── Controller 3 +``` + +其中一个 Controller 是 active controller,负责处理元数据变更: + +```plain +创建 Topic +删除 Topic +分区 Leader 选举 +Broker 上下线 +配置变更 +ACL 变更 +``` + +这些元数据变更会写入 Kafka 内部的元数据日志: + +```plain +__cluster_metadata +``` + +Controller quorum 通过 Raft 复制这份日志,只有多数派确认后,元数据变更才算提交。Broker 会从 Controller 拉取或接收最新元数据,然后据此处理客户端请求。 + +**** + +**和 ZooKeeper 模式的区别** + +ZooKeeper 模式: + +```plain +Kafka Broker + ZooKeeper +元数据存在 ZooKeeper +Controller 通过 ZooKeeper 协调 +``` + +KRaft 模式: + +```plain +Kafka Broker + Kafka Controller Quorum +元数据存在 Kafka 内部 Raft 日志 +Controller 由 Kafka 自己选举和管理 +``` + +**需要注意** + ++ KRaft 是新 Kafka 架构的主流方向。 ++ 生产环境建议 Controller 节点使用奇数个,比如 3 或 5。 ++ Controller quorum 需要多数派可用,否则元数据变更不可用。 ++ Broker 和 Controller 可以混合部署,也可以分离部署。 ++ 较大生产集群通常建议 Broker 和 Controller 分离。 ++ KRaft 模式下常见关键配置: + +```plain +process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@host1:9093,2@host2:9093,3@host3:9093 +controller.listener.names=CONTROLLER +``` + +一句话总结: + +**KRaft 是 Kafka 用 Raft 协议实现的内置元数据管理机制,用来替代 ZooKeeper,让 Kafka 自己完成 Controller 选举、元数据复制、分区管理和故障恢复。** + +# Metadata +Kafka 元数据就是集群的“路由表 + 状态表”。 + +它告诉客户端和 Broker: + +1. **有哪些 Topic 和 Partition** +2. **每个 Partition 的 Leader 是谁** +3. **每个 Partition 有哪些副本、ISR 是哪些** +4. **Broker 列表和地址是什么** +5. **Topic/集群配置是什么** +6. **ACL、配额等管理信息是什么** + +客户端必须先拿到元数据,才知道消息该发到哪个 Broker、从哪个 Broker 拉取。 + +**元数据的原理** + +Producer / Consumer 启动时会连接 `bootstrap.servers` 中的任意 Broker,请求集群元数据: + +```plain +Client -> 任意 Broker -> 返回 Metadata +``` + +拿到元数据后,客户端就知道: + +```plain +Topic A / Partition 0 的 Leader 是 broker-1 +Topic A / Partition 1 的 Leader 是 broker-2 +``` + +然后客户端直接连接对应 Leader 读写: + +```plain +Producer -> Partition Leader +Consumer <- Partition Leader +``` + +如果发生变化,比如: + +```plain +Leader 切换 +Broker 上下线 +Topic 新增/删除 +Partition 增加 +``` + +客户端会刷新元数据,重新找到正确的 Leader。 + +在服务端,元数据由 **Controller** 管理: + +```plain +ZooKeeper 模式:元数据主要存 ZooKeeper,Controller 负责协调 +KRaft 模式:元数据存 Kafka 内部 Raft 日志,由 Controller Quorum 管理 +``` + +现代 Kafka 的 KRaft 模式中,元数据变更会写入内部元数据日志: + +```plain +__cluster_metadata +``` + +Controller quorum 通过 Raft 复制并提交这些变更,Broker 再同步最新元数据。 + +**需要注意** + ++ `bootstrap.servers` 只是入口,不是完整路由配置。 ++ 真正决定客户端连接地址的是 `advertised.listeners`。 ++ 客户端会缓存元数据,不是每次请求都查。 ++ Leader 变化后,客户端可能短暂收到 `NOT_LEADER_OR_FOLLOWER`,然后刷新元数据重试。 ++ 元数据异常常见表现是:客户端连得上 bootstrap,但无法生产/消费。 ++ KRaft 模式下,Controller quorum 不可用会影响元数据变更,但已知 Leader 的普通读写不一定立刻中断。 + +一句话总结: + +**Kafka 元数据负责描述集群拓扑、Topic/Partition 分布、Leader 副本和配置状态,是客户端路由、Broker 协作、Leader 选举和集群管理的基础。** + +# ACK +Kafka 中的 ACK 是 **Producer 写消息时要求 Broker 返回的确认级别**。 + +它主要决定: + +1. **消息写入成功的判定标准** +2. **可靠性和吞吐之间的取舍** +3. **Producer 是否需要等待副本同步完成** + +**ACK 的原理** + +Producer 发送消息到 Partition Leader 后,Broker 根据 `acks` 配置决定什么时候返回成功。 + +常见配置: + +```plain +acks=0 +acks=1 +acks=all +``` + +含义: + +```plain +acks=0 +Producer 不等 Broker 响应。 +吞吐最高,可靠性最低,消息可能还没到 Broker 就丢。 + +acks=1 +Leader 写入本地日志后就返回成功。 +性能较好,但如果 Leader 宕机且 Follower 还没同步,消息可能丢。 + +acks=all +Leader 等待 ISR 中足够副本确认后再返回成功。 +可靠性最高,但延迟更高。 +``` + +`acks=all` 通常还要配合: + +```plain +min.insync.replicas=2 +replication.factor=3 +``` + +含义是: + +```plain +3 个副本 +至少 2 个 ISR 副本确认 +Producer 才认为写入成功 +``` + +如果 ISR 数量不足,Producer 会收到失败,而不是冒险写入。 + +**需要注意** + ++ `acks=all` 不是等所有副本,而是等 ISR 中满足条件的副本。 ++ `acks=1` 只保证 Leader 写入,不保证 Follower 已复制。 ++ `acks=0` 无法知道消息是否成功,失败也不容易重试。 ++ 强可靠场景建议: + +```plain +acks=all +enable.idempotence=true +retries=2147483647 +``` + ++ 可靠性还依赖服务端: + +```plain +min.insync.replicas=2 +unclean.leader.election.enable=false +``` + +一句话总结: + +**ACK 是 Kafka Producer 控制写入确认语义的核心配置;**`**acks=0**`** 追求吞吐,**`**acks=1**`** 折中,**`**acks=all**`** 配合 ISR 和 **`**min.insync.replicas**`** 提供更强可靠性。** + +# Segment +Kafka Segment 是 **Partition 日志文件的分段存储单元**。 + +它主要解决: + +1. **避免单个日志文件过大** +Partition 是无限追加日志,如果只用一个文件会越来越大,难以管理。 +2. **提升清理效率** +Kafka 可以按 Segment 删除过期数据,而不是逐条删除消息。 +3. **加快查找** +每个 Segment 配套索引文件,可以快速根据 offset 或时间定位消息。 + +**Segment 的原理** + +一个 Partition 底层由多个 Segment 组成: + +```plain +Partition-0 + ├── 00000000000000000000.log + ├── 00000000000000000000.index + ├── 00000000000000000000.timeindex + ├── 00000000000000001000.log + ├── 00000000000000001000.index + └── 00000000000000001000.timeindex +``` + +每个 Segment 文件名是该 Segment 的 **base offset**。 + +例如: + +```plain +00000000000000001000.log +``` + +表示这个 Segment 从 offset `1000` 附近开始存消息。 + +Kafka 写入时,只追加到当前 active Segment。达到一定大小或时间后,会滚动生成新的 Segment: + +```plain +segment.bytes +segment.ms +``` + +查询消息时: + +```plain +1. 根据 offset 找到对应 Segment +2. 通过 .index 定位物理文件位置 +3. 从 .log 文件读取消息 +``` + +**相关文件** + +```plain +.log 存储真实消息数据 +.index offset -> 文件物理位置索引 +.timeindex timestamp -> offset 索引 +``` + +**需要注意** + ++ Segment 是 Partition 内部存储结构,不是 Kafka 对外概念。 ++ Kafka 删除过期数据通常以 Segment 为单位。 ++ active Segment 一般不会被删除,只有滚动后的旧 Segment 才会按策略清理。 ++ `segment.bytes` 越小,清理更及时,但文件更多。 ++ `segment.bytes` 越大,文件更少,但过期数据释放可能不够及时。 ++ Log Compaction 也主要围绕 Segment 做清理和重写。 + +一句话总结: + +**Segment 是 Kafka 将 Partition 日志切分成多个文件的机制,用于支撑顺序追加写、快速索引查询、按时间/大小滚动以及高效日志清理。** + +# Index文件 +Kafka 的 index 文件是 **Segment 的稀疏索引文件**,用于快速定位消息在 `.log` 文件中的物理位置。 + +它主要解决: + +1. **避免全量扫描 log 文件** +根据 offset 快速找到消息大概位置。 +2. **提升消费和查找效率** +Consumer 从某个 offset 开始消费时,可以快速定位。 +3. **支持时间查询** +根据时间戳查找对应 offset。 + +**Index 文件类型** + +一个 Segment 通常有几类相关文件: + +```plain +.log 真实消息数据 +.index offset 索引 +.timeindex 时间索引 +``` + +核心是: + +```plain +.index: relative offset -> physical position +.timeindex: timestamp -> relative offset +``` + +**Index 文件原理** + +Kafka 的 `.index` 不是每条消息都建索引,而是 **稀疏索引**。 + +例如 Segment base offset 是 `1000`: + +```plain +00000000000000001000.log +00000000000000001000.index +``` + +`.index` 中记录的是相对 offset: + +```plain +relative offset = message offset - base offset +``` + +示例: + +```plain +offset 1000 -> relative offset 0 +offset 1050 -> relative offset 50 +offset 1100 -> relative offset 100 +``` + +索引项大致保存: + +```plain +relative offset -> log 文件中的物理 position +``` + +查找 offset `1080` 时: + +```plain +1. 根据 offset 找到对应 Segment +2. 在 .index 中二分查找 <= 1080 的最大索引项 +3. 拿到对应 physical position +4. 从 .log 文件该位置顺序扫描少量消息 +5. 找到目标 offset +``` + +因为是稀疏索引,所以最终还需要扫一小段 log,但不用从 Segment 开头扫。 + +**timeindex 原理** + +`.timeindex` 用于按时间查找消息。 + +它保存类似: + +```plain +timestamp -> relative offset +``` + +当 Consumer 使用按时间定位,比如 `offsetsForTimes()` 时,Kafka 会通过 `.timeindex` 找到接近该时间的 offset,再结合 `.index` 和 `.log` 定位消息。 + +**需要注意** + ++ index 文件是 Segment 的辅助文件。 ++ `.index` 加速 offset 查找。 ++ `.timeindex` 加速 timestamp 查找。 ++ Kafka index 是稀疏索引,不是每条消息一条索引。 ++ 稀疏索引节省空间,但查找后还需顺序扫描少量消息。 ++ 索引损坏时 Kafka 可以基于 `.log` 文件重建索引。 + +一句话总结: + +**Kafka index 文件是 Segment 的稀疏索引,通过 offset 或 timestamp 快速定位 **`**.log**`** 文件中的物理位置,在节省索引空间的同时提升消息查找和消费定位效率。** + From 11980650ffe30604d9ee66ede0a7365435d22753 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sun, 5 Jul 2026 20:58:16 +0800 Subject: [PATCH 16/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=96=87=E7=AB=A0?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index abff78b..7ca38a0 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - kafka源码分析 - kafka版本:4.2 - - [kafka 核心概念扫描]() + - [kafka 核心概念扫描](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%E6%A0%B8%E5%BF%83%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) From b11b27aee3f62e501c029f98ed9c0347aad0e6f9 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 7 Jul 2026 23:25:39 +0800 Subject: [PATCH 17/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 9 +- ...37\347\220\206\346\200\273\347\273\223.md" | 1865 +++++++++++++++++ 2 files changed, 1873 insertions(+), 1 deletion(-) create mode 100644 "note/kafka/kafka\346\234\200\346\226\260\347\211\210\346\234\254\346\240\270\345\277\203\346\246\202\345\277\265\344\270\216\346\240\270\345\277\203\345\216\237\347\220\206\346\200\273\347\273\223.md" diff --git a/README.md b/README.md index 7ca38a0..2dcbc08 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 # 目录 - kafka源码分析 - - kafka版本:4.2 + - kafka版本:4.3 - [kafka 核心概念扫描](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%E6%A0%B8%E5%BF%83%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) @@ -71,6 +71,13 @@ Java相关流行框架源码分析,学习以及总结,项目持续更新中 - [kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) - [kafka ISR原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) +重平衡 +![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) +![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) + +ISR/HW/LEO关系图 +![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) + - JDK源码学习 - JDK版本:1.8.0_77 - [深入学习String源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) diff --git "a/note/kafka/kafka\346\234\200\346\226\260\347\211\210\346\234\254\346\240\270\345\277\203\346\246\202\345\277\265\344\270\216\346\240\270\345\277\203\345\216\237\347\220\206\346\200\273\347\273\223.md" "b/note/kafka/kafka\346\234\200\346\226\260\347\211\210\346\234\254\346\240\270\345\277\203\346\246\202\345\277\265\344\270\216\346\240\270\345\277\203\345\216\237\347\220\206\346\200\273\347\273\223.md" new file mode 100644 index 0000000..8a4052f --- /dev/null +++ "b/note/kafka/kafka\346\234\200\346\226\260\347\211\210\346\234\254\346\240\270\345\277\203\346\246\202\345\277\265\344\270\216\346\240\270\345\277\203\345\216\237\347\220\206\346\200\273\347\273\223.md" @@ -0,0 +1,1865 @@ ++ 转载请标明出处 + + + +本章旨在快速扫盲\回顾 Apache Kafka 最新版本最核心、最重要的概念以及原理,不做具体框架源码级深入分析。 + +截至本文编写时间,Apache Kafka 官网下载页显示最新支持版本为 **Kafka 4.3.1**,发布日期为 **2026-06-25**。Kafka 4.x 之后的核心变化是 **ZooKeeper 架构退出主线,KRaft 成为 Kafka 元数据管理和集群控制的核心模式**。 + +参考来源: + ++ Apache Kafka Downloads: https://kafka.apache.org/community/downloads/ ++ Apache Kafka 4.3 Documentation: https://kafka.apache.org/43/ ++ Apache Kafka Documentation: https://kafka.apache.org/documentation/ + +重点覆盖: + ++ Kafka 是什么 ++ Topic、Partition、Replica、Broker ++ Producer、Consumer、Consumer Group ++ Offset、Log、Segment、Index ++ ISR、Leader、Follower、高水位 HW ++ Controller、KRaft、Metadata Quorum ++ 生产者发送流程、ACK、幂等、事务 ++ 消费者拉取流程、位移提交、Rebalance ++ 传统 Consumer Group 和新版 Consumer 协议 ++ Share Group / Queues for Kafka ++ 顺序性、可靠性、可用性、一致性 ++ Page Cache、顺序写、零拷贝 ++ Log Retention、Log Compaction ++ Tiered Storage ++ Kafka Connect、Kafka Streams ++ 安全机制、监控指标、常见问题排查 ++ 工程选型和实践建议 + +# Kafka +Kafka 是 **一个分布式事件流平台,用于高吞吐、低延迟、可持久化地发布、存储、订阅和处理事件数据**。 + +它主要解决: + +1. **系统解耦** +生产者只把消息写入 Kafka,不需要直接调用所有下游系统。 +2. **削峰填谷** +流量高峰先写入 Kafka,下游按自身能力消费。 +3. **数据持久化** +消息不是消费后立即删除,而是按保留策略存储一段时间。 +4. **实时数据流处理** +适合日志、埋点、监控、交易事件、CDC、实时计算。 +5. **多订阅方消费** +同一份数据可以被多个消费者组独立消费。 + +**传统同步调用** + +```plain +订单服务 + -> 库存服务 + -> 积分服务 + -> 风控服务 + -> 推荐服务 +``` + +调用链长,任何下游慢都会影响上游。 + +**使用 Kafka** + +```plain +订单服务 + -> 写入 order_created Topic + +库存服务消费 +积分服务消费 +风控服务消费 +推荐服务消费 +``` + +各系统通过事件解耦。 + +**需要注意** + ++ Kafka 不是传统意义上“发完即删”的简单消息队列。 ++ Kafka 的核心抽象是分布式日志。 ++ Kafka 强在高吞吐、可扩展、可回放、多消费者订阅。 ++ Kafka 不适合所有低延迟 RPC 场景,也不替代数据库事务。 ++ Kafka 4.x 新集群应优先理解 KRaft,而不是旧 ZooKeeper 架构。 + +一句话总结: + +**Kafka 是以分布式日志为核心的事件流平台,通过 Topic、Partition、Offset 和 Consumer Group 实现高吞吐、可持久化、可回放的数据流。** + +# Topic +Topic 是 **Kafka 中消息的逻辑分类,相当于一类事件流的名字**。 + +它主要解决: + +1. **按业务分类消息** +2. **隔离不同事件流** +3. **让生产者和消费者通过名称解耦** +4. **作为权限、保留策略、分区配置的管理单位** + +**例子** + +```plain +order_created +payment_success +user_login +stock_changed +app_log +``` + +**生产和消费** + +```plain +Producer -> order_created Topic +Consumer <- order_created Topic +``` + +**Topic 不是物理上的一个文件** + +Kafka 内部会把 Topic 拆成多个 Partition: + +```plain +order_created + -> partition-0 + -> partition-1 + -> partition-2 +``` + +**需要注意** + ++ Topic 命名要稳定,避免随意变更。 ++ Topic 数量过多会增加元数据和运维成本。 ++ Topic 的分区数、保留时间、清理策略要结合业务设计。 ++ 不同业务语义的数据不要随意混入同一个 Topic。 + +一句话总结: + +**Topic 是 Kafka 消息的逻辑分类,真正承载数据和并行能力的是 Topic 下的 Partition。** + +# Partition +Partition 是 **Topic 的物理分片,也是 Kafka 并行、顺序和扩展能力的基础单位**。 + +它主要解决: + +1. **单个 Topic 水平扩展** +2. **提升读写吞吐** +3. **让消费者并行消费** +4. **保证分区内消息顺序** + +**结构** + +```plain +Topic: order_created + +partition-0: + msg0, msg1, msg2 + +partition-1: + msg0, msg1, msg2 + +partition-2: + msg0, msg1, msg2 +``` + +**分区内有序** + +```plain +partition-0: + offset 0 + offset 1 + offset 2 +``` + +Kafka 只保证: + +```plain +同一个 Partition 内有序 +``` + +不保证: + +```plain +整个 Topic 全局有序 +``` + +**分区选择** + +```plain +有 key: + 根据 key hash 选择分区 + +无 key: + 按生产者分区策略分配 + +自定义: + 业务实现 Partitioner +``` + +**需要注意** + ++ 分区数决定消费者组内最大并行度。 ++ 分区数不是越多越好,过多会增加文件、网络和元数据成本。 ++ 扩容分区会影响 key 到分区的映射,可能破坏同 key 顺序。 ++ 需要同一业务实体有序时,应让同一 key 进入同一分区。 + +一句话总结: + +**Partition 是 Kafka 并行和顺序的核心单位,Kafka 保证分区内有序,不保证 Topic 全局有序。** + +# Broker +Broker 是 **Kafka 集群中的服务节点,负责接收生产请求、存储分区日志、处理消费请求和复制数据**。 + +它主要解决: + +1. **承载 Topic Partition 数据** +2. **处理客户端读写请求** +3. **参与副本复制** +4. **向集群汇报状态** +5. **作为 KRaft 节点参与元数据管理,取决于角色配置** + +**集群结构** + +```plain +Broker-1 + -> topicA partition-0 leader + -> topicB partition-1 follower + +Broker-2 + -> topicA partition-1 leader + -> topicA partition-0 follower + +Broker-3 + -> topicA partition-2 leader + -> topicB partition-0 follower +``` + +**Broker 角色** + +```plain +普通 Broker: + 处理数据读写 + +Controller: + 管理集群元数据和分区状态 + +Combined 节点: + 同时承担 Broker 和 Controller 角色 +``` + +**需要注意** + ++ Broker 宕机会触发副本 Leader 切换。 ++ 单个 Broker 磁盘、网络、CPU 都可能成为瓶颈。 ++ Broker 不是无状态服务,磁盘数据和 broker.id/node.id 很关键。 ++ 生产环境要合理规划磁盘、网络、机架和副本分布。 + +一句话总结: + +**Broker 是 Kafka 集群的数据服务节点,负责消息读写、日志存储、副本复制,并在 KRaft 架构中配合集群元数据管理。** + +# Replica +Replica 是 **Partition 的副本,用于提升 Kafka 数据可靠性和可用性**。 + +它主要解决: + +1. **Broker 宕机后的数据可用** +2. **降低单点故障风险** +3. **支持 Leader 故障切换** +4. **提升数据持久性** + +**副本结构** + +```plain +partition-0 + leader: Broker-1 + follower: Broker-2 + follower: Broker-3 +``` + +**Leader 副本** + +```plain +处理生产者写入 +处理消费者读取 +维护分区日志主流程 +``` + +**Follower 副本** + +```plain +从 Leader 拉取数据 +保持日志同步 +在 Leader 故障时可能被选为新 Leader +``` + +**副本因子** + +```plain +replication.factor = 3 +``` + +表示每个 Partition 有 3 个副本。 + +**需要注意** + ++ 副本越多,可靠性越好,但存储和复制成本越高。 ++ 生产常见副本因子是 3。 ++ Follower 默认不对普通消费者提供读服务。 ++ 副本分布要避免多个副本落在同一故障域。 + +一句话总结: + +**Replica 通过 Leader/Follower 副本复制提升 Kafka 的可靠性和可用性,生产环境通常使用 3 副本。** + +# ISR +ISR 是 **In-Sync Replicas,同步副本集合,表示当前跟得上 Leader 的副本列表**。 + +它主要解决: + +1. **判断哪些副本是健康同步副本** +2. **控制消息提交安全性** +3. **Leader 故障时选择可靠新 Leader** +4. **配合 acks 和 min.insync.replicas 保证写入可靠性** + +**结构** + +```plain +partition-0 + leader: Broker-1 + ISR: [Broker-1, Broker-2, Broker-3] +``` + +如果 Broker-3 落后太多: + +```plain +ISR: [Broker-1, Broker-2] +``` + +**写入确认** + +```plain +acks=all +min.insync.replicas=2 +``` + +含义: + +```plain +消息至少要被 ISR 中足够数量副本确认 +才认为写入成功 +``` + +**需要注意** + ++ ISR 不是固定副本列表,会动态变化。 ++ ISR 缩小通常表示副本复制延迟或故障。 ++ min.insync.replicas 设置过低会降低可靠性。 ++ 设置过高会降低可用性,副本不足时写入失败。 + +一句话总结: + +**ISR 表示当前与 Leader 保持同步的副本集合,是 Kafka 判断写入可靠性和故障切换安全性的核心机制。** + +# Offset +Offset 是 **消息在 Partition 内的递增位置编号**。 + +它主要解决: + +1. **标识消息位置** +2. **支持消费者断点续读** +3. **支持消息回放** +4. **让不同消费者组独立维护消费进度** + +**结构** + +```plain +partition-0: + offset 0 -> msgA + offset 1 -> msgB + offset 2 -> msgC +``` + +**消费者进度** + +```plain +consumer group A: + partition-0 committed offset = 2 + +consumer group B: + partition-0 committed offset = 0 +``` + +同一个 Topic 可以被多个消费者组独立消费。 + +**Offset 提交** + +```plain +消费消息 + -> 处理业务 + -> 提交 offset +``` + +**需要注意** + ++ Offset 只在单个 Partition 内有意义。 ++ 提交 offset 表示消费者认为之前的数据已经处理完成。 ++ 先提交再处理可能丢消息。 ++ 先处理再提交可能重复消费。 ++ Kafka 通常要求业务端做好幂等处理。 + +一句话总结: + +**Offset 是 Partition 内消息位置,消费者通过提交 Offset 记录消费进度,从而支持断点续读和消息回放。** + +# Log +Kafka Log 是 **Partition 在磁盘上的追加写日志,每条消息按 Offset 顺序追加到日志末尾**。 + +它主要解决: + +1. **消息持久化** +2. **顺序写入** +3. **按 Offset 查询** +4. **支持消息保留和回放** + +**日志结构** + +```plain +topic-partition/ + 00000000000000000000.log + 00000000000000000000.index + 00000000000000000000.timeindex + 00000000000000100000.log + 00000000000000100000.index + 00000000000000100000.timeindex +``` + +**追加写** + +```plain +Producer 写入 + -> Broker 追加到当前 active segment + -> 分配 offset + -> 等待确认 +``` + +**读取** + +```plain +Consumer 请求 offset + -> Broker 通过 index 定位 segment + -> 从 log 文件顺序读取 + -> 返回消息批次 +``` + +**需要注意** + ++ Kafka 不是随机更新消息,而是追加日志。 ++ 日志文件按 Segment 切分,便于删除、压缩和索引。 ++ 磁盘顺序写是 Kafka 高吞吐的重要原因。 ++ 日志保留时间到期后,旧消息会被清理,不能无限回放。 + +一句话总结: + +**Kafka 的核心存储模型是 Partition 追加日志,消息按 Offset 顺序写入磁盘,并通过 Segment 和索引支持高效读写。** + +# Segment 和 Index +Segment 是 **Kafka 把 Partition 日志切成多个文件段的机制**。 + +Index 是 **Kafka 用于快速定位 Offset 或时间戳对应日志位置的稀疏索引**。 + +它主要解决: + +1. **避免单个日志文件过大** +2. **方便按时间或大小清理旧数据** +3. **提升 Offset 定位效率** +4. **支持快速查找时间戳附近消息** + +**Segment 文件** + +```plain +00000000000000000000.log +00000000000000000000.index +00000000000000000000.timeindex +``` + +文件名前缀表示: + +```plain +该 Segment 起始 offset +``` + +**索引定位** + +```plain +Consumer 请求 offset=12345 + -> 找到起始 offset <= 12345 的 segment + -> 查 offset index 找近似位置 + -> 从 log 文件顺序扫描到目标消息 +``` + +**需要注意** + ++ Kafka 使用稀疏索引,不是每条消息都有索引。 ++ Segment 越大,文件数量少,但清理粒度更粗。 ++ Segment 越小,清理更灵活,但文件数量更多。 ++ 索引损坏通常可以根据 log 文件重建。 + +一句话总结: + +**Segment 把分区日志拆成可管理文件段,Index 用稀疏索引快速定位消息位置,是 Kafka 高效存储和清理的基础。** + +# Producer +Producer 是 **向 Kafka Topic 写入消息的客户端**。 + +它主要解决: + +1. **序列化业务数据** +2. **选择目标 Topic 和 Partition** +3. **批量发送消息** +4. **处理重试、幂等和事务** +5. **接收 Broker 写入确认** + +**发送流程** + +```plain +业务调用 send() + -> 序列化 key/value/header + -> 分区器选择 partition + -> 写入 RecordAccumulator + -> Sender 线程批量发送 + -> Broker 写入日志 + -> 返回 ack +``` + +**重要参数** + +```plain +acks: + 写入确认级别 + +batch.size: + 批次大小 + +linger.ms: + 等待聚合批次时间 + +compression.type: + 压缩方式 + +retries: + 重试次数 + +enable.idempotence: + 幂等生产 +``` + +**需要注意** + ++ batch.size 和 linger.ms 影响吞吐和延迟。 ++ 压缩可以降低网络和磁盘成本,但增加 CPU 开销。 ++ 重试可能导致乱序,幂等生产可以降低重复写入风险。 ++ 生产者发送成功不等于所有消费者已经消费。 + +一句话总结: + +**Producer 通过序列化、分区、批量、压缩、重试和 ACK 机制把业务事件高效可靠地写入 Kafka。** + +# ACK +ACK 是 **生产者写入消息时要求 Broker 返回确认的级别**。 + +它主要解决: + +1. **控制写入可靠性** +2. **平衡吞吐和数据安全** +3. **配合副本同步策略** + +**acks=0** + +```plain +Producer 不等待 Broker 确认 +吞吐高 +可能丢消息 +``` + +**acks=1** + +```plain +Leader 写入成功后返回 +性能较好 +Leader 宕机且 Follower 未同步时可能丢消息 +``` + +**acks=all** + +```plain +Leader 等待 ISR 副本满足要求后返回 +可靠性最高 +延迟更高 +``` + +**常见可靠配置** + +```plain +acks=all +min.insync.replicas=2 +replication.factor=3 +enable.idempotence=true +``` + +**需要注意** + ++ acks=all 还需要合理设置 min.insync.replicas。 ++ min.insync.replicas 过低会降低副本确认意义。 ++ 副本不足时,acks=all 可能导致写入失败。 ++ 高可靠配置会牺牲部分可用性和延迟。 + +一句话总结: + +**ACK 决定生产者等待到什么程度才认为写入成功,acks=all 配合 ISR 和 min.insync.replicas 是高可靠写入的核心。** + +# 幂等生产者 +幂等生产者是 **Kafka 通过 Producer ID、Producer Epoch 和 Sequence Number 避免生产者重试导致重复写入的机制**。 + +它主要解决: + +1. **网络超时后重试可能重复写入** +2. **Broker 已写入但 ACK 丢失** +3. **单分区内重试乱序和重复** + +**问题场景** + +```plain +Producer 发送 msg-1 + -> Broker 写入成功 + -> ACK 在网络中丢失 + -> Producer 重试 msg-1 + -> 可能重复写入 +``` + +**幂等机制** + +```plain +每个 Producer 有 PID +每个分区维护递增 sequence +Broker 检查 sequence 是否连续 +重复 sequence 被识别并丢弃 +``` + +**需要注意** + ++ 幂等生产者主要保证单 Producer 会话内、单分区的幂等写入。 ++ 它不等于业务幂等。 ++ 应用重启、业务重放、消费者重复处理仍要业务幂等。 ++ 跨分区、跨 Topic 的原子写入需要事务。 + +一句话总结: + +**幂等生产者通过 PID 和序列号避免发送重试造成的重复写入,是 Kafka 精确一次语义的重要基础。** + +# Kafka 事务 +Kafka 事务是 **让生产者把多条消息、多个分区写入和消费位移提交作为一个原子单元提交或回滚的机制**。 + +它主要解决: + +1. **跨分区原子写入** +2. **消费-处理-生产链路一致性** +3. **Exactly Once 语义基础** +4. **避免下游读到未提交事务消息** + +**典型流程** + +```plain +initTransactions +beginTransaction + -> consume input records + -> produce output records + -> sendOffsetsToTransaction +commitTransaction +``` + +失败时: + +```plain +abortTransaction +``` + +**读隔离级别** + +```plain +read_uncommitted: + 可以读到未提交事务消息 + +read_committed: + 只读已提交事务消息 +``` + +**需要注意** + ++ Kafka 事务解决 Kafka 内部读写链路的一致性,不自动保证外部数据库事务。 ++ 事务会增加协调成本和延迟。 ++ transactional.id 必须稳定且唯一。 ++ EOS 仍然要求业务端正确处理外部副作用。 + +一句话总结: + +**Kafka 事务把多分区写入和消费位移提交纳入同一原子单元,是 Kafka Exactly Once 处理链路的核心机制。** + +# Consumer +Consumer 是 **从 Kafka Topic 拉取并处理消息的客户端**。 + +它主要解决: + +1. **按 Offset 拉取消息** +2. **维护消费进度** +3. **参与消费者组分区分配** +4. **处理重试、提交和回放** + +**消费流程** + +```plain +Consumer 加入 Consumer Group + -> 获取分区分配 + -> poll 拉取消息 + -> 执行业务处理 + -> 提交 offset +``` + +**拉模式** + +Kafka 消费是客户端主动拉取: + +```plain +Consumer -> Broker: fetch request +Broker -> Consumer: records +``` + +优点: + +```plain +消费者按自身能力拉取 +容易做批量 +方便控制进度 +``` + +**需要注意** + ++ poll 要持续调用,否则可能被认为消费者失活。 ++ 消费处理慢会导致消费堆积。 ++ 自动提交 offset 简单但容易产生丢失或重复风险。 ++ 业务处理必须考虑重复消费。 + +一句话总结: + +**Consumer 通过 poll 主动拉取消息,并用 Offset 管理消费进度;消费可靠性核心在于处理和提交 Offset 的顺序。** + +# Consumer Group +Consumer Group 是 **Kafka 实现消费负载均衡和多订阅方隔离的机制**。 + +它主要解决: + +1. **同组消费者分摊分区消费** +2. **不同组独立消费同一 Topic** +3. **消费者故障后自动接管分区** +4. **支持水平扩展消费能力** + +**同组消费** + +```plain +Topic: order_created + partition-0 + partition-1 + partition-2 + +Group-A: + consumer-1 -> partition-0 + consumer-2 -> partition-1 + consumer-3 -> partition-2 +``` + +同一个消费者组内: + +```plain +一个 Partition 同一时刻通常只分配给一个 Consumer +``` + +不同消费者组: + +```plain +Group-A 可以消费一遍 +Group-B 也可以独立消费一遍 +``` + +**需要注意** + ++ 同一组消费者数量超过分区数时,多出来的消费者会空闲。 ++ 分区数决定同组最大并行消费数。 ++ 消费者上下线会触发 Rebalance。 ++ 不同业务系统通常应该使用不同 group.id。 + +一句话总结: + +**Consumer Group 让同一组消费者分摊分区消费,让不同组可以独立订阅同一份 Kafka 数据。** + +# Offset 提交 +Offset 提交是 **消费者把已处理到的位置记录到 Kafka 内部 Topic 中的机制**。 + +它主要解决: + +1. **消费者重启后从上次位置继续** +2. **消费者组故障恢复** +3. **消费进度可观测** +4. **支持手动回放和重置** + +**自动提交** + +```plain +enable.auto.commit=true +``` + +特点: + +```plain +使用简单 +可能处理前就提交 +异常时可能丢消息 +``` + +**手动提交** + +```plain +处理成功后 commitSync 或 commitAsync +``` + +常见流程: + +```plain +poll records + -> 处理业务 + -> 处理成功 + -> commit offset +``` + +**重复和丢失** + +```plain +先提交,再处理: + 处理失败会丢消息 + +先处理,再提交: + 提交失败会重复消费 +``` + +**需要注意** + ++ Kafka 默认更容易做到至少一次。 ++ 想避免业务重复,必须实现幂等。 ++ offset 提交的是下一条要消费的位置。 ++ 提交粒度过细影响性能,过粗增加重复范围。 + +一句话总结: + +**Offset 提交决定消费者故障恢复位置,可靠消费通常选择处理成功后再提交,并通过业务幂等接受重复消费。** + +# Rebalance +Rebalance 是 **消费者组成员或订阅分区变化时,重新分配 Partition 的过程**。 + +它主要解决: + +1. **消费者扩缩容** +2. **消费者故障接管** +3. **分区新增后的重新分配** +4. **消费负载均衡** + +**触发条件** + +```plain +消费者加入组 +消费者离开组 +消费者心跳超时 +Topic 分区变化 +订阅 Topic 变化 +``` + +**传统流程** + +```plain +暂停消费 + -> 组协调器确认成员 + -> 执行分区分配 + -> 通知消费者新分配 + -> 恢复消费 +``` + +**问题** + +```plain +消费暂停 +重复消费风险 +分区频繁迁移 +消费延迟抖动 +``` + +**优化方向** + +```plain +静态成员 +协作式再均衡 +新版消费者组协议 +合理 session.timeout.ms 和 max.poll.interval.ms +``` + +**需要注意** + ++ Rebalance 不是错误,但频繁 Rebalance 是问题。 ++ 处理时间超过 max.poll.interval.ms 可能被踢出消费者组。 ++ 消费者扩缩容会带来短暂消费抖动。 ++ 需要监控 Rebalance 次数和消费延迟。 + +一句话总结: + +**Rebalance 负责消费者组分区重新分配,是消费弹性和故障接管的基础,但频繁发生会造成延迟和重复消费风险。** + +# 新版 Consumer 协议 +新版 Consumer 协议是 **Kafka 近年来对消费者组协调和 Rebalance 流程的改进方向,目标是降低协调成本、减少客户端复杂度和提升 Rebalance 稳定性**。 + +它主要解决: + +1. **传统 Rebalance 停顿时间较长** +2. **客户端承担过多分配逻辑** +3. **成员变化时影响范围较大** +4. **大规模消费者组协调成本高** + +**传统模型** + +```plain +Consumer Leader 负责分区分配 +Group Coordinator 管理成员 +分配结果再同步给组成员 +``` + +**新版思路** + +```plain +Broker 侧承担更多协调职责 +协议更适合增量变化 +减少全组停顿 +降低客户端分配复杂度 +``` + +**需要注意** + ++ 不同 Kafka 客户端版本对新协议支持程度不同。 ++ 升级前要确认 broker、client、配置兼容性。 ++ 新协议优化 Rebalance,但不消除业务处理慢导致的消费堆积。 ++ 老版本消费者组仍然可能继续使用传统协议。 + +一句话总结: + +**新版 Consumer 协议把消费者组协调能力进一步服务端化,目标是减少大规模消费者组 Rebalance 的停顿和复杂度。** + +# Share Group +Share Group 是 **Kafka 4.x 引入的队列式消费能力方向,也常被称为 Queues for Kafka,用于让多个消费者共享处理同一分区中的不同消息**。 + +它主要解决: + +1. **传统消费者组并行度受分区数限制** +2. **队列场景下不希望过度增加分区** +3. **多个消费者共同处理同一分区消息** +4. **消息级确认和重投递需求** + +**传统 Consumer Group** + +```plain +一个 Partition 同一时刻分配给一个 Consumer +消费者数量超过分区数会空闲 +``` + +**Share Group** + +```plain +多个 Consumer 可以共享同一 Partition 的消息处理 +更接近队列语义 +支持消息级别的处理跟踪 +``` + +**适合场景** + +```plain +任务队列 +异步作业处理 +消费者数量动态变化 +不要求严格分区顺序的工作负载 +``` + +**需要注意** + ++ Share Group 更适合队列式任务,不是替代所有 Consumer Group。 ++ 如果业务强依赖分区内顺序,仍要谨慎使用。 ++ 客户端和 Broker 版本都要确认支持。 ++ 消息级确认也要求业务处理幂等。 + +一句话总结: + +**Share Group 让 Kafka 支持更接近队列的消费模型,突破传统消费者组并行度受分区数限制的问题。** + +# KRaft +KRaft 是 **Kafka Raft Metadata mode,用 Kafka 自身的 Raft 协议管理集群元数据,替代旧 ZooKeeper 架构**。 + +它主要解决: + +1. **去除 ZooKeeper 外部依赖** +2. **统一 Kafka 元数据管理** +3. **提升控制面扩展性和恢复速度** +4. **简化部署和运维** + +**旧架构** + +```plain +Broker + -> ZooKeeper 保存元数据 + -> Controller 监听和管理集群状态 +``` + +**KRaft 架构** + +```plain +Controller Quorum + -> 使用 Raft 管理元数据日志 + -> Broker 从 Controller 获取元数据 +``` + +**角色** + +```plain +broker: + 处理数据读写 + +controller: + 管理元数据、分区状态、Leader 选举 + +broker,controller: + 同一进程同时承担两种角色 +``` + +**需要注意** + ++ Kafka 4.x 新集群应按 KRaft 理解和部署。 ++ Controller Quorum 的节点数和可靠性非常关键。 ++ 元数据日志也需要持久化和备份意识。 ++ 从旧 ZooKeeper 集群迁移要严格遵循版本和迁移流程。 + +一句话总结: + +**KRaft 用 Kafka 自身的 Raft 元数据仲裁替代 ZooKeeper,是 Kafka 4.x 集群控制面的核心架构。** + +# Controller +Controller 是 **Kafka 集群控制面核心角色,负责管理元数据、分区 Leader、Broker 状态和集群变更**。 + +它主要解决: + +1. **分区 Leader 选举** +2. **Broker 上下线处理** +3. **Topic 和 Partition 元数据管理** +4. **副本状态变更** +5. **集群控制事件传播** + +**KRaft Controller Quorum** + +```plain +Controller-1 +Controller-2 +Controller-3 +``` + +通过 Raft 维护元数据日志: + +```plain +metadata log + -> topic 创建 + -> partition leader 变更 + -> broker 注册 + -> 配置变更 +``` + +**Broker 获取元数据** + +```plain +Broker 启动 + -> 注册到 Controller + -> 接收元数据变更 + -> 根据分区角色处理读写和复制 +``` + +**需要注意** + ++ Controller 不直接承载普通消息数据读写,主要负责控制面。 ++ Controller Quorum 不稳定会影响集群管理能力。 ++ Controller 与 Broker 的网络、磁盘、时间配置都要稳定。 ++ 小集群可以 combined 模式,大规模生产更建议角色隔离。 + +一句话总结: + +**Controller 是 Kafka 控制面核心,KRaft 模式下通过 Controller Quorum 和元数据日志管理整个集群状态。** + +# 高水位 HW +高水位 HW 是 **High Watermark,表示消费者可见的最高已提交 Offset 边界**。 + +它主要解决: + +1. **防止消费者读到未复制安全的数据** +2. **定义已提交消息范围** +3. **支持副本故障恢复** +4. **保证消费者读取的一致性** + +**简化示例** + +```plain +Leader log end offset = 10 +Follower-1 log end offset = 10 +Follower-2 log end offset = 8 + +HW = 8 +``` + +消费者只能读到: + +```plain +offset < HW +``` + +**为什么需要 HW** + +如果消费者读到 Leader 上还没复制到足够副本的数据: + +```plain +Leader 宕机 +新 Leader 没有这条消息 +消费者之前读到的数据消失 +``` + +HW 避免这种不一致。 + +**需要注意** + ++ HW 推进依赖副本复制进度。 ++ 副本落后会导致 HW 推进变慢。 ++ 消费延迟不只和消费者有关,也可能和副本同步有关。 ++ 事务场景还会涉及 LSO 等读可见边界。 + +一句话总结: + +**HW 定义消费者可见的已提交日志边界,确保消费者不会读到可能在故障切换中丢失的未安全复制数据。** + +# 顺序性 +Kafka 顺序性是 **Kafka 保证同一 Partition 内消息按 Offset 顺序追加和读取,但不保证 Topic 全局顺序**。 + +它主要解决: + +1. **同一业务实体事件有序** +2. **状态变更按顺序处理** +3. **避免乱序更新导致状态错误** + +**保证同 key 有序** + +```plain +key = orderId + +order_created +order_paid +order_shipped +``` + +通过 key hash 进入同一分区: + +```plain +partition-3: + order_created + order_paid + order_shipped +``` + +**无法天然保证** + +```plain +不同 Partition 之间的全局顺序 +多个 Producer 之间的严格全局顺序 +扩分区后的同 key 历史映射不变 +``` + +**需要注意** + ++ 需要有序的业务必须设计 key。 ++ 消费端并发处理同一分区消息也可能破坏业务顺序。 ++ 重试、异步处理、死信补偿都可能引入乱序。 ++ 全局有序通常意味着单分区,吞吐和可用性会受限。 + +一句话总结: + +**Kafka 只保证分区内顺序,业务要通过合理 key 设计把同一实体事件路由到同一分区。** + +# 可靠性 +Kafka 可靠性是 **消息从生产、存储、复制到消费处理过程中尽量不丢失、可恢复、可重试的能力**。 + +它主要依赖: + +```plain +replication.factor +acks +min.insync.replicas +ISR +幂等生产者 +事务 +手动提交 offset +消费者幂等 +``` + +**高可靠生产** + +```plain +replication.factor=3 +min.insync.replicas=2 +acks=all +enable.idempotence=true +``` + +**高可靠消费** + +```plain +关闭自动提交 +处理成功后提交 offset +业务幂等 +失败重试 +死信或补偿 +``` + +**需要注意** + ++ Kafka 能降低消息丢失概率,但不能替业务处理外部副作用兜底。 ++ 消费端提交 offset 和业务数据库事务之间仍可能不一致。 ++ 高可靠配置会增加延迟并降低部分故障场景下可用性。 ++ 消息至少一次通常比恰好一次更常见,业务幂等必须做。 + +一句话总结: + +**Kafka 可靠性来自副本、ACK、ISR、幂等、事务和正确的 Offset 提交,最终仍需要业务幂等和补偿闭环。** + +# Exactly Once +Exactly Once 是 **Kafka 在特定读写 Kafka 的处理链路中,通过幂等生产者和事务实现的一次性处理语义**。 + +它主要解决: + +1. **生产重试导致重复写入** +2. **消费后生产再提交 Offset 的一致性** +3. **流处理拓扑中重复输出** + +**典型场景** + +```plain +input-topic + -> Kafka Streams 处理 + -> output-topic +``` + +事务保证: + +```plain +输出消息 + + 输入 offset 提交 +作为一个事务提交 +``` + +**需要注意** + ++ Exactly Once 通常限定在 Kafka 到 Kafka 的处理链路内。 ++ 如果写外部数据库、调用 HTTP 接口,仍要外部系统支持幂等或事务。 ++ EOS 会带来额外协调开销。 ++ 不要把 Kafka EOS 理解成所有业务副作用绝对只发生一次。 + +一句话总结: + +**Kafka Exactly Once 主要保证 Kafka 内部消费-处理-生产链路的一致性,外部系统仍需要幂等和事务配合。** + +# Page Cache +Page Cache 是 **Linux 用内存缓存文件数据的机制,也是 Kafka 高吞吐的重要基础**。 + +它主要解决: + +1. **减少磁盘读写延迟** +2. **让顺序写更高效** +3. **利用操作系统缓存热点日志** +4. **降低 JVM 堆内缓存压力** + +**写入流程** + +```plain +Broker 追加消息 + -> 写入 OS Page Cache + -> 后台刷盘 +``` + +**读取流程** + +```plain +Consumer 拉取消息 + -> 如果日志在 Page Cache + -> 直接从内存读取 + -> 否则触发磁盘读取 +``` + +**为什么 Kafka 不把消息都放 JVM 堆** + +```plain +避免 GC 压力 +利用 OS Page Cache +支持大数据量日志缓存 +``` + +**需要注意** + ++ Kafka 写入返回不一定等于立刻 fsync 到磁盘。 ++ Page Cache 被挤压会影响消费读取性能。 ++ Broker 内存规划要给 Page Cache 留空间。 ++ 不要只看 JVM 堆,还要看系统可用内存和磁盘缓存。 + +一句话总结: + +**Kafka 高吞吐大量依赖 Linux Page Cache,合理的内存规划应给操作系统缓存留出足够空间。** + +# 顺序写和零拷贝 +顺序写和零拷贝是 **Kafka 高吞吐的重要底层优化思想**。 + +**顺序写** + +Kafka 追加日志: + +```plain +只在文件末尾追加 +避免大量随机写 +充分利用磁盘顺序写能力 +``` + +**零拷贝** + +消费发送数据时: + +```plain +磁盘文件 + -> Page Cache + -> Socket + -> 网卡 +``` + +减少: + +```plain +内核态到用户态拷贝 +用户态到内核态拷贝 +CPU 拷贝成本 +``` + +**典型机制** + +```plain +sendfile +transferTo +``` + +**需要注意** + ++ 零拷贝不是完全没有拷贝,而是减少不必要的数据拷贝。 ++ 压缩、加密、消息转换可能影响零拷贝路径。 ++ 顺序写不代表磁盘永远不是瓶颈。 ++ 网络带宽常常比磁盘更早成为 Kafka 瓶颈。 + +一句话总结: + +**Kafka 通过追加顺序写和零拷贝减少磁盘随机 IO 与 CPU 拷贝开销,从而获得高吞吐。** + +# Log Retention +Log Retention 是 **Kafka 按时间或大小保留消息日志,超过保留策略后清理旧 Segment 的机制**。 + +它主要解决: + +1. **控制磁盘空间** +2. **支持一段时间内消息回放** +3. **让 Kafka 成为持久事件日志** +4. **自动清理过期数据** + +**常见配置** + +```plain +retention.ms +retention.bytes +segment.ms +segment.bytes +``` + +**清理流程** + +```plain +Segment 达到保留条件 + -> 被标记为可删除 + -> 后台清理线程删除 +``` + +**需要注意** + ++ Retention 是按 Segment 粒度清理,不是逐条消息。 ++ 消费者太慢,消息可能在消费前被清理。 ++ 磁盘容量规划必须结合写入速率和保留时间。 ++ Retention 不等于归档,长期留存可考虑 Tiered Storage。 + +一句话总结: + +**Log Retention 通过时间和大小策略清理旧日志,是 Kafka 控制磁盘空间和支持有限回放窗口的核心机制。** + +# Log Compaction +Log Compaction 是 **Kafka 按 key 保留最新值、清理旧值的日志压缩机制**。 + +它主要解决: + +1. **保存最新状态** +2. **支持状态重建** +3. **减少相同 key 的历史冗余** +4. **适合变更日志和配置类数据** + +**普通日志** + +```plain +key=user1 value=A +key=user1 value=B +key=user1 value=C +``` + +压缩后至少保留: + +```plain +key=user1 value=C +``` + +**删除语义** + +```plain +key=user1 value=null +``` + +称为 tombstone,用于表示删除。 + +**适合场景** + +```plain +用户最新状态 +配置变更 +CDC upsert 事件 +状态存储 changelog +``` + +**需要注意** + ++ Compaction 不保证立刻清理。 ++ 同一个 key 的旧值在一段时间内仍可能存在。 ++ key 不能为空,否则无法按 key 压缩。 ++ 不适合需要完整历史审计的场景。 + +一句话总结: + +**Log Compaction 按 key 保留最新状态,适合状态重建和变更日志,不适合完整历史事件审计。** + +# Tiered Storage +Tiered Storage 是 **Kafka 把较旧日志从本地磁盘分层存储到远端存储的能力,用于降低本地磁盘压力并扩大保留窗口**。 + +它主要解决: + +1. **本地磁盘容量有限** +2. **长时间保留消息成本高** +3. **历史数据回放需求** +4. **冷热数据分层** + +**基本思路** + +```plain +热数据: + 保留在 Broker 本地磁盘 + +冷数据: + 上传到远端对象存储或远端存储系统 +``` + +消费旧数据时: + +```plain +Consumer 请求旧 offset + -> Broker 从远端读取 + -> 返回给 Consumer +``` + +**需要注意** + ++ Tiered Storage 会引入远端存储延迟和成本。 ++ 历史回放性能通常不如本地热数据。 ++ 远端存储可靠性和权限也要纳入运维。 ++ 要区分本地保留策略和远端保留策略。 + +一句话总结: + +**Tiered Storage 通过冷热分层把旧日志放到远端存储,扩大 Kafka 数据保留窗口并降低本地磁盘压力。** + +# Kafka Connect +Kafka Connect 是 **Kafka 的数据集成框架,用于把外部系统数据导入 Kafka,或把 Kafka 数据导出到外部系统**。 + +它主要解决: + +1. **数据库 CDC 接入** +2. **日志和文件采集** +3. **写入搜索、数仓、对象存储** +4. **减少重复开发导入导出程序** + +**Source Connector** + +```plain +外部系统 + -> Connector + -> Kafka Topic +``` + +例如: + +```plain +MySQL CDC -> Kafka +``` + +**Sink Connector** + +```plain +Kafka Topic + -> Connector + -> 外部系统 +``` + +例如: + +```plain +Kafka -> Elasticsearch +Kafka -> S3 +``` + +**需要注意** + ++ Connector 也要考虑 offset、重试、幂等和死信。 ++ Source 和 Sink 的一致性语义取决于外部系统能力。 ++ 大规模 Connect 集群要关注任务分配和 Rebalance。 ++ Connector 配置错误可能导致重复写入或数据延迟。 + +一句话总结: + +**Kafka Connect 用标准化 Connector 把 Kafka 与数据库、搜索、数仓、对象存储等系统连接起来,是数据集成的重要组件。** + +# Kafka Streams +Kafka Streams 是 **Kafka 官方 Java 流处理库,用于基于 Kafka Topic 构建实时处理应用**。 + +它主要解决: + +1. **流式转换** +2. **聚合和窗口** +3. **Join** +4. **状态ful 处理** +5. **Exactly Once 流处理** + +**基本模型** + +```plain +input topic + -> filter/map/groupBy/window/join + -> output topic +``` + +**状态存储** + +```plain +本地 State Store + -> changelog topic + -> 故障后从 changelog 恢复 +``` + +**常见场景** + +```plain +实时统计 +风控规则 +订单状态聚合 +用户行为窗口计算 +流表 Join +``` + +**需要注意** + ++ Kafka Streams 是库,不是独立集群计算框架。 ++ 应用实例数量和 Topic 分区数影响并行度。 ++ 状态存储需要磁盘和恢复时间规划。 ++ 复杂大规模计算可以评估 Flink、Spark Streaming 等框架。 + +一句话总结: + +**Kafka Streams 让应用直接基于 Kafka 构建实时流处理拓扑,适合轻量到中等复杂度的流式计算。** + +# 安全机制 +Kafka 安全机制包括 **认证、授权、加密和审计相关能力**。 + +它主要解决: + +1. **客户端身份确认** +2. **Topic 访问控制** +3. **网络传输加密** +4. **多租户隔离** +5. **操作审计** + +**常见能力** + +```plain +SSL/TLS: + 加密传输和证书认证 + +SASL: + 用户认证机制 + +ACL: + 资源访问控制 + +Principal: + 客户端身份 + +Authorizer: + 授权判断 +``` + +**ACL 示例语义** + +```plain +用户 userA + -> 允许写 Topic order_created + -> 允许读 Group inventory-service +``` + +**需要注意** + ++ 生产环境不要裸奔明文无认证。 ++ ACL 要按最小权限配置。 ++ 证书和密码要有轮换机制。 ++ 多租户 Kafka 要同时做配额、ACL 和 Topic 命名规范。 + +一句话总结: + +**Kafka 安全依赖 TLS、SASL、ACL 和配额等机制,核心目标是确认身份、限制权限和保护数据传输。** + +# 监控指标 +Kafka 监控是 **围绕 Broker、Topic、Partition、Producer、Consumer、Controller 和系统资源建立可观测性的过程**。 + +重点关注: + +```plain +Broker 存活 +Controller 状态 +Under Replicated Partitions +Offline Partitions +ISR 变化 +请求延迟 +生产吞吐 +消费吞吐 +Consumer Lag +磁盘使用率 +网络吞吐 +Page Cache 命中 +GC +``` + +**Consumer Lag** + +```plain +Lag = 最新 offset - 已提交 offset +``` + +表示: + +```plain +消费者落后多少消息 +``` + +**副本健康** + +```plain +Under Replicated Partitions > 0: + 有副本落后或不可用 + +Offline Partitions > 0: + 有分区不可服务 +``` + +**需要注意** + ++ 只看 Broker 是否存活不够。 ++ Lag 高可能是消费者慢,也可能是分区不均、下游慢、Rebalance 频繁。 ++ 磁盘满会直接影响 Broker 稳定性。 ++ Controller 和元数据仲裁健康在 KRaft 模式下非常重要。 + +一句话总结: + +**Kafka 监控要同时关注吞吐、延迟、Lag、副本健康、Controller、磁盘、网络和 JVM,不能只看进程存活。** + +# 常见问题排查 +Kafka 问题通常涉及客户端、Broker、网络、磁盘、Controller、消费者组和业务处理多个层面。 + +**消息堆积** + +排查方向: + +```plain +Consumer Lag 是否持续增长 +消费者实例数是否不足 +分区数是否限制并行度 +业务处理是否变慢 +下游数据库是否慢 +是否频繁 Rebalance +``` + +**生产延迟升高** + +排查方向: + +```plain +acks 配置是否变更 +ISR 是否缩小 +Broker 请求队列是否堆积 +磁盘 IO 是否变慢 +网络是否打满 +batch 和 linger 是否合理 +``` + +**频繁 Rebalance** + +排查方向: + +```plain +消费者是否频繁重启 +poll 间隔是否超过 max.poll.interval.ms +session.timeout.ms 是否太短 +GC 是否过长 +网络是否抖动 +``` + +**副本不同步** + +排查方向: + +```plain +Follower 网络是否慢 +Broker 磁盘 IO 是否慢 +复制线程是否繁忙 +Leader 写入压力是否过高 +跨机房复制是否延迟大 +``` + +**消息重复消费** + +排查方向: + +```plain +是否处理成功但 offset 提交失败 +是否 Rebalance 后重复拉取 +是否生产者重试重复写 +业务是否缺少幂等键 +``` + +**需要注意** + ++ 先区分生产端、Broker 端、消费端、下游业务端。 ++ Kafka 层成功不代表业务处理成功。 ++ 消息重复是常态风险,业务必须幂等。 ++ 排查要看时间线:配置变更、发布、流量峰值、Broker 事件。 + +一句话总结: + +**Kafka 排查要从生产、存储、复制、消费、下游和控制面逐层定位,Lag、ISR、Rebalance、磁盘和网络是最常见线索。** + +# 工程选型 +Kafka 适合高吞吐、可持久化、可回放的事件流场景,但不是所有异步通信都适合 Kafka。 + +**适合 Kafka** + +```plain +日志采集 +埋点数据 +CDC 数据流 +事件驱动架构 +实时计算输入 +多系统订阅同一事件 +高吞吐异步处理 +数据回放 +``` + +**不适合 Kafka** + +```plain +极低延迟 RPC +小规模简单任务队列 +强事务同步调用 +复杂延迟调度 +大量单条消息精细 ACK 的传统队列场景 +``` + +**选型关注** + +```plain +吞吐量 +消息保留时间 +顺序要求 +可靠性要求 +消费并行度 +运维能力 +客户端生态 +监控和告警 +``` + +**需要注意** + ++ Kafka 强项是事件流和日志,不是简单 RPC 替代品。 ++ 分区数、副本数、保留时间要在上线前规划。 ++ 高可靠要接受更高延迟和更复杂配置。 ++ 上线 Kafka 必须配套监控、容量规划和故障演练。 + +一句话总结: + +**Kafka 最适合高吞吐、可持久化、可回放、多订阅的事件流场景,选型时要重点评估顺序、可靠性、延迟、容量和运维成本。** + +# 总结 +Kafka 的核心不是简单消息队列,而是一个围绕分布式日志构建的事件流平台。 + +常见理解路径: + +```plain +基础模型: + Topic、Partition、Replica、Broker + +写入链路: + Producer、Partitioner、Batch、ACK、ISR、幂等、事务 + +存储模型: + Log、Segment、Index、Page Cache、顺序写、零拷贝 + +消费模型: + Consumer、Consumer Group、Offset、Rebalance、Share Group + +控制面: + KRaft、Controller、Metadata Quorum + +可靠性: + 副本、ISR、HW、acks、min.insync.replicas、事务 + +生态: + Kafka Connect、Kafka Streams、Tiered Storage + +运维: + Lag、ISR、Controller、磁盘、网络、GC、Rebalance +``` + +一句话总结: + +**Kafka 的本质是分布式、可复制、可持久化、可回放的日志系统;真正用好 Kafka,要理解分区日志、消费者组、副本同步、KRaft 控制面和端到端可靠性。** From 8a4c0aae19b709618efe9289f2ff17e38ea5d621 Mon Sep 17 00:00:00 2001 From: CoderBruis <37364336+coderbruis@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:27:09 +0800 Subject: [PATCH 18/37] Update README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 更新kafka版本 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2dcbc08..9113c62 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ rocketmq - rocketmq + rocketmq

From c1c2302d3598e69ce42711d7868ad7498b259741 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 7 Jul 2026 23:30:48 +0800 Subject: [PATCH 19/37] =?UTF-8?q?=E4=BC=98=E5=8C=96readme=E5=B8=83?= =?UTF-8?q?=E5=B1=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 251 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 142 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 2dcbc08..7ba7704 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ ![JavaSourceCodeLearningImage](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/images/JavaSourceCodeLearningImage.png) +

Java Source Code Learning

+ +

+ 一份面向 Java 后端工程师的源码阅读地图:从 JDK / JUC 到 Spring、Netty、Kafka、RocketMQ,按核心链路拆解框架设计与底层实现。 +

+

JDK @@ -20,139 +26,166 @@ Netty - rocketmq + RocketMQ - rocketmq + Kafka

-Java相关流行框架源码分析,学习以及总结,项目持续更新中。 +## 项目亮点 -框架或者源码包括: +| 你能看到什么 | 重点能力 | +| --- | --- | +| JDK / JUC 源码 | 集合、并发、线程池、内存模型、CompletableFuture | +| Spring / SpringBoot 源码 | IOC 容器、事件机制、启动流程、自动装配、扩展点 | +| SpringAOP / Security / OAuth2 | 代理机制、过滤器链、安全认证授权主流程 | +| Netty 源码 | Reactor 模型、EventLoop、ChannelPipeline、网络通信链路 | +| Kafka / RocketMQ 源码 | Broker、Producer、Consumer、Rebalance、消息存储与复制 | -✅ JDK源码 +## 学习路线 -✅ JUC源码 +```text +JDK / JUC 基础源码 + ↓ +Spring 容器与扩展点 + ↓ +SpringBoot 启动与自动装配 + ↓ +Netty 网络通信模型 + ↓ +Kafka / RocketMQ 消息系统源码 +``` -✅ Spring源码 +## 为什么要读源码? -✅ SpringBoot源码 +源码学习不是为了记住每一行实现,而是为了把框架背后的设计选择看清楚。 -✅ SpringAOP源码 +当线上问题出现时,读过核心链路的人通常能更快定位边界、判断根因、验证假设;当自己做系统设计时,也更容易借鉴成熟框架在模块拆分、扩展点设计、并发控制、性能优化上的经验。 -✅ SpringSecurity源码 +这个项目会围绕主流 Java 后端框架的核心路径持续整理源码分析、学习笔记和关键图解。 -✅ SpringSecurity OAuth2源码 +## 精选图解 -✅ JDK源码 +### Kafka Rebalance -✅ Dubbo源码 +![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) -✅ Netty源码 +![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) -✅ RocketMQ源码 +### Kafka ISR / HW / LEO -✅ kafka源码 +![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) -> 为什么要分析、学习源码? +## 内容目录 - 学习框架源码不仅能帮助我们在实际问题出现时快速定位问题、理解根因并高效解决,还能深入掌握框架的整体架构设计思路与核心设计模式,从而提升自身的系统设计能力与架构思维。 -同时,通过学习优秀开源框架的底层实现,可以不断强化对复杂系统拆分、模块协作以及性能优化的理解,这对于个人技术能力的长期成长至关重要。因此,源码学习虽然过程相对枯燥,但这是提升架构设计能力的必经路径。只有持续积累,才能在实际系统设计与工程实践中做到真正的“游刃有余”。 +
+Kafka 源码分析 + +- Kafka 版本:4.3 +- [Kafka 核心概念扫盲](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%E6%A0%B8%E5%BF%83%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) +- [Kafka Broker 核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) +- [Kafka Broker 源码分析:生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) +- [Kafka 消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) +- [Kafka Rebalance 核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) +- [Kafka ISR 原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) -# 目录 +
-- kafka源码分析 - - kafka版本:4.3 - - [kafka 核心概念扫描](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%E6%A0%B8%E5%BF%83%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - - [kafka broker核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - - [kafka broker核心源码分析——生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - - [kafka消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) - - [kafka Rebalance核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) - - [kafka ISR原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) +
+JDK / JUC 源码学习 -重平衡 -![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) -![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) +- JDK 版本:1.8.0_77 +- [深入学习 String 源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) +- [深入学习 String 源码与底层(二)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%BA%8C%EF%BC%89.md) +- [深入解读 CompletableFuture 源码与原理](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E8%AF%BBCompletableFuture%E6%BA%90%E7%A0%81%E4%B8%8E%E5%8E%9F%E7%90%86.md) +- [深入分析 ThreadLocal](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%88%86%E6%9E%90ThreadLocal.md) +- [深入学习 Java volatile 关键字](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Java%20volatile%E5%85%B3%E9%94%AE%E5%AD%97.md) +- [深入学习 Thread 底层原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Thread%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81.md) +- 深入学习 JDK1.7 / 1.8 HashMap 扩容原理 +- [开源项目里那些看不懂的位运算分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E5%BC%80%E6%BA%90%E9%A1%B9%E7%9B%AE%E9%87%8C%E9%82%A3%E4%BA%9B%E7%9C%8B%E4%B8%8D%E6%87%82%E7%9A%84%E4%BD%8D%E8%BF%90%E7%AE%97%E5%88%86%E6%9E%90.md) +- [ThreadPoolExecutor 源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E6%9E%90ThreadPoolExecutor%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) -ISR/HW/LEO关系图 -![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) +
+ +
+Spring 源码学习 + +- Spring 版本:5.2.1.RELEASE +- [深入 Spring 源码系列(一):在 IDEA 中构建 Spring 源码](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%E5%9C%A8IDEA%E4%B8%AD%E6%9E%84%E5%BB%BASpring%E6%BA%90%E7%A0%81.md) +- [深入 Spring 容器源码与时序图(上)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%E6%B7%B1%E5%85%A5Spring%E5%AE%B9%E5%99%A8%EF%BC%8C%E9%80%9A%E8%BF%87%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB%E5%92%8C%E6%97%B6%E5%BA%8F%E5%9B%BE%E6%9D%A5%E5%BD%BB%E5%BA%95%E5%BC%84%E6%87%82Spring%E5%AE%B9%E5%99%A8%EF%BC%88%E4%B8%8A%EF%BC%89.md) +- [深入 Spring 容器源码与时序图(下)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%E6%B7%B1%E5%85%A5Spring%E5%AE%B9%E5%99%A8%EF%BC%8C%E9%80%9A%E8%BF%87%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB%E5%92%8C%E6%97%B6%E5%BA%8F%E5%9B%BE%E6%9D%A5%E5%BD%BB%E5%BA%95%E5%BC%84%E6%87%82Spring%E5%AE%B9%E5%99%A8%EF%BC%88%E4%B8%8B%EF%BC%89.md) +- [深入 Spring 源码系列(补充篇):程序调用 Spring 源码](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E8%A1%A5%E5%85%85%E7%AF%87%EF%BC%89%E2%80%94%E2%80%94%E7%A8%8B%E5%BA%8F%E8%B0%83%E7%94%A8Spring%E6%BA%90%E7%A0%81.md) +- [从 Spring 源码中学习策略模式](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E4%BB%8ESpring%E6%BA%90%E7%A0%81%E4%B8%AD%E5%AD%A6%E4%B9%A0%E2%80%94%E2%80%94%E7%AD%96%E7%95%A5%E6%A8%A1%E5%BC%8F.md) + +
+ +
+SpringAOP 源码学习 + +- Spring 版本:5.2.1.RELEASE +- [深入学习 SpringAOP 源码(一):注册 AnnotationAwareAspectJAutoProxyCreator](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%E6%B3%A8%E5%86%8CAnnotationAwareAspectJAutoProxyCreator.md) +- [深入学习 SpringAOP 源码(二):深入 AnnotationAwareAspectJAutoProxyCreator](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20%E6%B7%B1%E5%85%A5AnnotationAwareAspectJAutoProxyCreator.md) +- [深入学习 SpringAOP 源码(三):揭开 JDK 动态代理和 CGLIB 代理的神秘面纱](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%B8%89%EF%BC%89%E2%80%94%E2%80%94%E6%8F%AD%E5%BC%80JDK%E5%8A%A8%E6%80%81%E4%BB%A3%E7%90%86%E5%92%8CCGLIB%E4%BB%A3%E7%90%86%E7%9A%84%E7%A5%9E%E7%A7%98%E9%9D%A2%E7%BA%B1.md) + +
+ +
+SpringBoot 源码学习 + +- SpringBoot 版本:2.2.1.RELEASE +- [深入浅出 SpringBoot 源码:SpringFactoriesLoader](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94SpringFactoriesLoader.md) +- [深入浅出 SpringBoot 源码:监听器与事件机制](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E7%9B%91%E5%90%AC%E5%99%A8%E4%B8%8E%E4%BA%8B%E4%BB%B6%E6%9C%BA%E5%88%B6.md) +- [深入浅出 SpringBoot 源码:系统初始化器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E7%B3%BB%E7%BB%9F%E5%88%9D%E5%A7%8B%E5%8C%96%E5%99%A8.md) +- [深入浅出 SpringBoot 源码:启动加载器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E5%90%AF%E5%8A%A8%E5%8A%A0%E8%BD%BD%E5%99%A8.md) + +
+ +
+SpringSecurity / OAuth2 源码学习 + +- SpringSecurity 版本:5.1.0.RELEASE +- [深入浅出 SpringSecurity 和 OAuth2(一):初识 SpringSecurity](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%20%E5%88%9D%E8%AF%86SpringSecurity.md) +- [深入浅出 SpringSecurity 和 OAuth2(二):安全过滤器 FilterChainProxy](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20%E5%AE%89%E5%85%A8%E8%BF%87%E6%BB%A4%E5%99%A8FilterChainProxy.md) +- [深入浅出 SpringSecurity 和 OAuth2(三):WebSecurity 建造核心逻辑](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%B8%89%EF%BC%89%E2%80%94%E2%80%94%20WebSecurity%E5%BB%BA%E9%80%A0%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91.md) +- [深入浅出 SpringSecurity 和 OAuth2(四):FilterChainProxy 过滤器链中的几个重要过滤器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E5%9B%9B%EF%BC%89%E2%80%94%E2%80%94%20FilterChainProxy%E8%BF%87%E6%BB%A4%E5%99%A8%E9%93%BE%E4%B8%AD%E7%9A%84%E5%87%A0%E4%B8%AA%E9%87%8D%E8%A6%81%E7%9A%84%E8%BF%87%E6%BB%A4%E5%99%A8.md) + +
+ +
+Netty 底层源码解析 + +- Netty 版本:4.1.43.Final +- [Netty 概念扫盲](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) +- [二进制运算以及源码、反码以及补码学习](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/%E4%BA%8C%E8%BF%9B%E5%88%B6.md) +- [Netty 源码包结构](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E6%BA%90%E7%A0%81%E5%8C%85%E7%BB%93%E6%9E%84.md) +- [Netty 底层源码解析:EventLoopGroup](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E4%B8%AD%E7%9A%84EventLoopGroup%E6%98%AF%E4%BB%80%E4%B9%88.md) +- [Netty 底层源码解析:初始 Netty 及其架构](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-%E5%88%9D%E5%A7%8BNetty%E5%8F%8A%E5%85%B6%E6%9E%B6%E6%9E%84.md) +- [Netty 底层源码解析:Netty 服务端启动分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-Netty%E6%9C%8D%E5%8A%A1%E7%AB%AF%E5%90%AF%E5%8A%A8%E5%88%86%E6%9E%90.md) +- [Netty 底层源码解析:NioEventLoop 原理分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-NioEventLoop%E5%8E%9F%E7%90%86%E5%88%86%E6%9E%90.md) +- [Netty 底层源码解析:ChannelPipeline 分析(上)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-ChannelPipeline%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%8A%EF%BC%89.md) +- [Netty 底层源码解析:ChannelPipeline 分析(下)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-ChannelPipeline%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%8B%EF%BC%89.md) +- [Netty 底层源码解析:NioServerSocketChannel 接受数据原理分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-NioServerSocketChannel%E6%8E%A5%E5%8F%97%E6%95%B0%E6%8D%AE%E5%8E%9F%E7%90%86%E5%88%86%E6%9E%90.md) +- Netty 底层源码解析:NioSocketChannel 接受、发送数据原理分析 +- Netty 底层源码解析:FastThreadLocal 原理分析 +- Netty 底层源码解析:内存分配原理分析 +- Netty 底层源码解析:RocketMQ 底层使用到的 Netty + +Netty 实战课相关代码位于 `Spring-Netty` 模块下的 `com/bruis/learnnetty/im` 包。 -- JDK源码学习 - - JDK版本:1.8.0_77 - - [深入学习String源码与底层(一)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%B8%80%EF%BC%89.md) - - [深入学习String源码与底层(二)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0String%E6%BA%90%E7%A0%81%E4%B8%8E%E5%BA%95%E5%B1%82%EF%BC%88%E4%BA%8C%EF%BC%89.md) - - [深入解读CompletableFuture源码与原理](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E8%AF%BBCompletableFuture%E6%BA%90%E7%A0%81%E4%B8%8E%E5%8E%9F%E7%90%86.md) - - [深入分析ThreadLocal](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%88%86%E6%9E%90ThreadLocal.md) - - [深入学习Java volatile关键字](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Java%20volatile%E5%85%B3%E9%94%AE%E5%AD%97.md) - - [深入学习Thread底层原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Thread%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81.md) - - [深入学习JDK1.7、8 HashMap扩容原理]() - - [开源项目里那些看不懂的位运算分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E5%BC%80%E6%BA%90%E9%A1%B9%E7%9B%AE%E9%87%8C%E9%82%A3%E4%BA%9B%E7%9C%8B%E4%B8%8D%E6%87%82%E7%9A%84%E4%BD%8D%E8%BF%90%E7%AE%97%E5%88%86%E6%9E%90.md) - - [ThreadPoolExecutor源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E6%9E%90ThreadPoolExecutor%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) - -- Spring源码学习 - - Spring版本:5.2.1.RELEASE - - - [深入Spring源码系列(一)——在IDEA中构建Spring源码](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%E5%9C%A8IDEA%E4%B8%AD%E6%9E%84%E5%BB%BASpring%E6%BA%90%E7%A0%81.md) - - [深入Spring源码系列(二)——深入Spring容器,通过源码阅读和时序图来彻底弄懂Spring容器(上)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%E6%B7%B1%E5%85%A5Spring%E5%AE%B9%E5%99%A8%EF%BC%8C%E9%80%9A%E8%BF%87%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB%E5%92%8C%E6%97%B6%E5%BA%8F%E5%9B%BE%E6%9D%A5%E5%BD%BB%E5%BA%95%E5%BC%84%E6%87%82Spring%E5%AE%B9%E5%99%A8%EF%BC%88%E4%B8%8A%EF%BC%89.md) - - [深入Spring源码系列(二)——深入Spring容器,通过源码阅读和时序图来彻底弄懂Spring容器(下)](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%E6%B7%B1%E5%85%A5Spring%E5%AE%B9%E5%99%A8%EF%BC%8C%E9%80%9A%E8%BF%87%E6%BA%90%E7%A0%81%E9%98%85%E8%AF%BB%E5%92%8C%E6%97%B6%E5%BA%8F%E5%9B%BE%E6%9D%A5%E5%BD%BB%E5%BA%95%E5%BC%84%E6%87%82Spring%E5%AE%B9%E5%99%A8%EF%BC%88%E4%B8%8B%EF%BC%89.md) - - [深入Spring源码系列(补充篇)——程序调用Spring源码](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E6%B7%B1%E5%85%A5Spring%E6%BA%90%E7%A0%81%E7%B3%BB%E5%88%97%EF%BC%88%E8%A1%A5%E5%85%85%E7%AF%87%EF%BC%89%E2%80%94%E2%80%94%E7%A8%8B%E5%BA%8F%E8%B0%83%E7%94%A8Spring%E6%BA%90%E7%A0%81.md) - - [从Spring源码中学习——策略模式](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Spring/%E4%BB%8ESpring%E6%BA%90%E7%A0%81%E4%B8%AD%E5%AD%A6%E4%B9%A0%E2%80%94%E2%80%94%E7%AD%96%E7%95%A5%E6%A8%A1%E5%BC%8F.md) - -- SpringAOP源码学习 - - Spring版本:5.2.1.RELEASE - - - [深入学习SpringAOP源码(一)——注册AnnotationAwareAspectJAutoProxyCreator](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%E6%B3%A8%E5%86%8CAnnotationAwareAspectJAutoProxyCreator.md) - - [深入学习SpringAOP源码(二)—— 深入AnnotationAwareAspectJAutoProxyCreator](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20%E6%B7%B1%E5%85%A5AnnotationAwareAspectJAutoProxyCreator.md) - - [深入学习SpringAOP源码(三)——揭开JDK动态代理和CGLIB代理的神秘面纱](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringAOP/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0SpringAOP%E6%BA%90%E7%A0%81%EF%BC%88%E4%B8%89%EF%BC%89%E2%80%94%E2%80%94%E6%8F%AD%E5%BC%80JDK%E5%8A%A8%E6%80%81%E4%BB%A3%E7%90%86%E5%92%8CCGLIB%E4%BB%A3%E7%90%86%E7%9A%84%E7%A5%9E%E7%A7%98%E9%9D%A2%E7%BA%B1.md) - -- SpringBoot源码学习 - - SpringBoot版本:2.2.1.RELEASE - - - [深入浅出SpringBoot源码——SpringFactoriesLoader](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94SpringFactoriesLoader.md) - - [深入浅出SpringBoot源码——监听器与事件机制](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E7%9B%91%E5%90%AC%E5%99%A8%E4%B8%8E%E4%BA%8B%E4%BB%B6%E6%9C%BA%E5%88%B6.md) - - [深入浅出SpringBoot源码——系统初始化器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E7%B3%BB%E7%BB%9F%E5%88%9D%E5%A7%8B%E5%8C%96%E5%99%A8.md) - - [深入浅出SpringBoot源码——启动加载器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringBoot/%E6%B7%B1%E5%85%A5SpringBoot%E6%BA%90%E7%A0%81%E5%AD%A6%E4%B9%A0%E4%B9%8B%E2%80%94%E2%80%94%E5%90%AF%E5%8A%A8%E5%8A%A0%E8%BD%BD%E5%99%A8.md) - -- SpringSecurity&OAuth2源码学习 - - SpringSecurity版本:5.1.0.RELEASE - - [深入浅出SpringSecurity和OAuth2(一)—— 初识SpringSecurity](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%B8%80%EF%BC%89%E2%80%94%E2%80%94%20%E5%88%9D%E8%AF%86SpringSecurity.md) - - [深入浅出SpringSecurity和OAuth2(二)—— 安全过滤器FilterChainProxy](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%BA%8C%EF%BC%89%E2%80%94%E2%80%94%20%E5%AE%89%E5%85%A8%E8%BF%87%E6%BB%A4%E5%99%A8FilterChainProxy.md) - - [深入浅出SpringSecurity和OAuth2(三)—— WebSecurity建造核心逻辑](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E4%B8%89%EF%BC%89%E2%80%94%E2%80%94%20WebSecurity%E5%BB%BA%E9%80%A0%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91.md) - - [深入浅出SpringSecurity和OAuth2(四)—— FilterChainProxy过滤器链中的几个重要的过滤器](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/SpringSecurity/%E4%BB%8E%E9%9B%B6%E5%BC%80%E5%A7%8B%E7%B3%BB%E7%BB%9F%E5%AD%A6%E4%B9%A0SpringSecurity%E5%92%8COAuth2%EF%BC%88%E5%9B%9B%EF%BC%89%E2%80%94%E2%80%94%20FilterChainProxy%E8%BF%87%E6%BB%A4%E5%99%A8%E9%93%BE%E4%B8%AD%E7%9A%84%E5%87%A0%E4%B8%AA%E9%87%8D%E8%A6%81%E7%9A%84%E8%BF%87%E6%BB%A4%E5%99%A8.md) - -- Netty底层源码解析 - - Netty版本:4.1.43.Final - - [Netty概念扫盲](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - - [二进制运算以及源码、反码以及补码学习](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/%E4%BA%8C%E8%BF%9B%E5%88%B6.md) - - [Netty源码包结构](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E6%BA%90%E7%A0%81%E5%8C%85%E7%BB%93%E6%9E%84.md) - - [Netty底层源码解析-EventLoopGroup](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E4%B8%AD%E7%9A%84EventLoopGroup%E6%98%AF%E4%BB%80%E4%B9%88.md) - - [Netty底层源码解析-初始Netty及其架构](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-%E5%88%9D%E5%A7%8BNetty%E5%8F%8A%E5%85%B6%E6%9E%B6%E6%9E%84.md) - - [Netty底层源码解析-Netty服务端启动分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-Netty%E6%9C%8D%E5%8A%A1%E7%AB%AF%E5%90%AF%E5%8A%A8%E5%88%86%E6%9E%90.md) - - [Netty底层源码解析-NioEventLoop原理分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-NioEventLoop%E5%8E%9F%E7%90%86%E5%88%86%E6%9E%90.md) - - [Netty底层源码解析-ChannelPipeline分析(上)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-ChannelPipeline%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%8A%EF%BC%89.md) - - [Netty底层源码解析-ChannelPipeline分析(下)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-ChannelPipeline%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%8B%EF%BC%89.md) - - [Netty底层源码解析-NioServerSocketChannel接受数据原理分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81%E8%A7%A3%E6%9E%90-NioServerSocketChannel%E6%8E%A5%E5%8F%97%E6%95%B0%E6%8D%AE%E5%8E%9F%E7%90%86%E5%88%86%E6%9E%90.md) - - Netty底层源码解析-NioSocketChannel接受、发送数据原理分析 - - Netty底层源码解析-FastThreadLocal原理分析 - - Netty底层源码解析-内存分配原理分析 - - Netty底层源码解析-RocketMQ底层使用到的Netty - -Netty实战课相关点位于:Spring-Netty,com/bruis/learnnetty/im包下,有需要的读者可前往查看。 - - -- RocketMQ底层源码解析 - - RocketMQ版本:4.9.0 - - RocketMQ底层源码解析-RocketMQ环境搭建 - - RocketMQ底层源码解析-本地调试RocketMQ源码 - - RocketMQ底层源码解析-NameServer分析 - - 持续更新中... - - - -# 支持 - - 原创不易,各位帅哥美女star支持下... +
+
+RocketMQ 底层源码解析 + +- RocketMQ 版本:4.9.0 +- RocketMQ 底层源码解析:RocketMQ 环境搭建 +- RocketMQ 底层源码解析:本地调试 RocketMQ 源码 +- RocketMQ 底层源码解析:NameServer 分析 +- 持续更新中... + +
+ +## 支持 +如果这个项目对你有帮助,欢迎 Star。源码学习是长期工程,我会持续补充更多核心链路分析、图解和实践示例。 From f64c6b7eab6b8086d80c54443f95e0f85d5839b2 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 7 Jul 2026 23:32:54 +0800 Subject: [PATCH 20/37] merged --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7ba7704..ceb6041 100644 --- a/README.md +++ b/README.md @@ -29,17 +29,17 @@ RocketMQ - Kafka + Kafka

## 项目亮点 -| 你能看到什么 | 重点能力 | -| --- | --- | -| JDK / JUC 源码 | 集合、并发、线程池、内存模型、CompletableFuture | -| Spring / SpringBoot 源码 | IOC 容器、事件机制、启动流程、自动装配、扩展点 | -| SpringAOP / Security / OAuth2 | 代理机制、过滤器链、安全认证授权主流程 | +| 你能看到什么 | 重点能力 | +| --- |--------------------------------------------| +| JDK / JUC 源码 | 集合、并发、线程池、内存模型、CompletableFuture | +| Spring / SpringBoot 源码 | IOC 容器、事件机制、启动流程、自动装配、扩展点 | +| SpringAOP / Security / OAuth2 | 代理机制、过滤器链、安全认证授权主流程 | | Netty 源码 | Reactor 模型、EventLoop、ChannelPipeline、网络通信链路 | | Kafka / RocketMQ 源码 | Broker、Producer、Consumer、Rebalance、消息存储与复制 | From 4c51768d5e0968484c61f7a041506bef41838a7e Mon Sep 17 00:00:00 2001 From: CoderBruis <37364336+coderbruis@users.noreply.github.com> Date: Wed, 8 Jul 2026 06:32:36 +0800 Subject: [PATCH 21/37] Update README.md --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index ceb6041..174d040 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -![JavaSourceCodeLearningImage](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/images/JavaSourceCodeLearningImage.png) -

Java Source Code Learning

From b41b9afef32065e2f2ddc2052cc22d399920438f Mon Sep 17 00:00:00 2001 From: coderbruis Date: Thu, 9 Jul 2026 22:39:05 +0800 Subject: [PATCH 22/37] =?UTF-8?q?=E8=A1=A5=E5=85=85kafka=E6=9E=B6=E6=9E=84?= =?UTF-8?q?=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 174d040..01817ba 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,9 @@ Kafka / RocketMQ 消息系统源码 ## 精选图解 +### Kafka架构图 +![kafka_architecture](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_architecture.png) + ### Kafka Rebalance ![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) From de5de4482c23c14a6d41269159433be351358d9b Mon Sep 17 00:00:00 2001 From: coderbruis Date: Fri, 10 Jul 2026 20:21:27 +0800 Subject: [PATCH 23/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0HashMap=E5=8E=9F?= =?UTF-8?q?=E7=90=86=E5=88=86=E6=9E=90=E6=96=87=E7=AB=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 32 +- ...20\347\240\201\345\210\206\346\236\220.md" | 630 ++++++++++++++++++ 2 files changed, 646 insertions(+), 16 deletions(-) create mode 100644 "note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" diff --git a/README.md b/README.md index 01817ba..5fffc49 100644 --- a/README.md +++ b/README.md @@ -63,21 +63,6 @@ Kafka / RocketMQ 消息系统源码 这个项目会围绕主流 Java 后端框架的核心路径持续整理源码分析、学习笔记和关键图解。 -## 精选图解 - -### Kafka架构图 -![kafka_architecture](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_architecture.png) - -### Kafka Rebalance - -![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) - -![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) - -### Kafka ISR / HW / LEO - -![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) - ## 内容目录

@@ -91,6 +76,18 @@ Kafka / RocketMQ 消息系统源码 - [Kafka Rebalance 核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) - [Kafka ISR 原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) +Kafka架构图 +![kafka_architecture](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_architecture.png) + +Kafka Rebalance流程图 +![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) + +![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) + +Kafka ISR / HW / LEO关系图 +![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) + +
@@ -103,10 +100,13 @@ Kafka / RocketMQ 消息系统源码 - [深入分析 ThreadLocal](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%88%86%E6%9E%90ThreadLocal.md) - [深入学习 Java volatile 关键字](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Java%20volatile%E5%85%B3%E9%94%AE%E5%AD%97.md) - [深入学习 Thread 底层原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Thread%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81.md) -- 深入学习 JDK1.7 / 1.8 HashMap 扩容原理 +- [深入学习HashMap 底层源码与原理]() - [开源项目里那些看不懂的位运算分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E5%BC%80%E6%BA%90%E9%A1%B9%E7%9B%AE%E9%87%8C%E9%82%A3%E4%BA%9B%E7%9C%8B%E4%B8%8D%E6%87%82%E7%9A%84%E4%BD%8D%E8%BF%90%E7%AE%97%E5%88%86%E6%9E%90.md) - [ThreadPoolExecutor 源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E6%9E%90ThreadPoolExecutor%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) +HashMap原理图 +![HashMap原理图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/HashMap.png) +
diff --git "a/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" new file mode 100644 index 0000000..61386e6 --- /dev/null +++ "b/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -0,0 +1,630 @@ +当前HashMap版本:JDK1.8 + +# HashMap的重要成员变量以及内部类 +> **默认容量:DEFAULT_INITIAL_CAPACITY** +> + +```java +static final int DEFAULT_INITIAL_CAPACITY = 1 << 4; // aka 16 +``` + +当HashMap没有设置大小时,调用HashMap的put方法时,会进行初始值,并使用DEFAULT_INITIAL_CAPACITY设置默认大小。调用位置在: + +```java +final Node[] resize() { + ... + else { // zero initial threshold signifies using defaults + // HashMap默认大小,16 + newCap = DEFAULT_INITIAL_CAPACITY; + // 扩容阈值,12 + newThr = (int)(DEFAULT_LOAD_FACTOR * DEFAULT_INITIAL_CAPACITY); + } + ... +} +``` + + + +> **扩容阈值:DEFAULT_LOAD_FACTOR** +> + +```java +static final float DEFAULT_LOAD_FACTOR = 0.75f; +``` + +在resize()中可以看到,当HashMap首次添加元素,调用put时,会计算第一次扩容阈值12,也就是说HashMap中元素=12即触发扩容。 + + + +扩容实际发生在putVal()中,源码如下: + +```java +final V putVal(int hash, K key, V value, boolean onlyIfAbsent, + boolean evict) { + ... + if (++size > threshold) + resize(); + ... +} +``` + +当HashMap中添加了新元素,size递增之后判断是否大于扩容阈值。 + + + +> **树化相关配置:TREEIFY_THRESHOLD、UNTREEIFY_THRESHOLD、MIN_TREEIFY_CAPACITY** +> + +```java +// 桶内链表长度达到 8,考虑红黑树化 +static final int TREEIFY_THRESHOLD = 8; +// 数组容量至少 64,才真正允许红黑树化 +static final int UNTREEIFY_THRESHOLD = 6; +// 红黑树节点减少到 6 个或更少,考虑退化回链表 +static final int MIN_TREEIFY_CAPACITY = 64; +``` + +链表长到 8 时,如果数组容量小于 64,先扩容;如果数组容量已经至少 64,才转红黑树。树节点少到 6 时,再退化回链表。 + + + +```java +final V putVal(int hash, K key, V value, boolean onlyIfAbsent, + boolean evict) { + ... + if (binCount >= TREEIFY_THRESHOLD - 1) // -1 for 1st + treeifyBin(tab, hash); + ... +} + +final void treeifyBin(Node[] tab, int hash) { + ... + if (tab == null || (n = tab.length) < MIN_TREEIFY_CAPACITY) + // 先扩容 + resize(); + else if ((e = tab[index = (n - 1) & hash]) != null) { + // 红黑树化 + } + ... +} +``` + +可以看到,当HashMap中元素小于MIN_TREEIFY_CAPACITY时,是先进行的扩容,而非直接红黑树化。 + + + +> **HashMap内部类Node** +> + +这个类就是HashMap桶中存储的基本单元类,HashMap其实可以理解为一个数组,数组里每个位置叫一个桶bucket。 + +```java +static class Node implements Map.Entry { + final int hash; + final K key; + V value; + Node next; +} +``` + +在HashMap中通过一个Node数组来存储Node节点。 + +```java +transient Node[] table; +``` + +如果多个 key 经过 hash 计算后落到同一个桶,就会通过 `next` 串起来,形成链表: + +```plain +table[3] + | + v +Node(key1, value1) + | + next + v +Node(key2, value2) + | + next + v +Node(key3, value3) +``` + +HashMap 的每个桶位保存一个头节点引用;发生 hash 冲突时,新节点通过 next 挂在这个桶的链表后面;链表过长时可能转成红黑树。 + +还有一个细节需要注意,Node不仅存了key、value最核心的键值对信息,还存储了这个Node的hash值,这个hash值的作用是:**用于快速比较、查找、扩容迁移和树化查找,避免反复计算 hashCode,也保证节点定位稳定。** + +hash核心作用代码: + +```java +final V putVal(int hash, K key, V value, boolean onlyIfAbsent, + boolean evict) { + ... + Node e; K k; + if (p.hash == hash && + ((k = p.key) == key || (key != null && key.equals(k)))) + e = p; + ... +} +``` + +putVal() 里比较 key 是否已经存在,这里先比较hash值,如果hash值不一样,则key一定不相等,可以直接跳过,避免频繁调用 equals()。 + + + +查找时也会用到hash: + +```java +final Node getNode(int hash, Object key) { + ... + if (first.hash == hash && // always check first node + ((k = first.key) == key || (key != null && key.equals(k)))) + return first; + ... +} +``` + +**总结:Node.hash 是 key 的缓存 hash 值,用于快速比较、查找、扩容迁移和树化查找,避免反复计算 hashCode,也保证节点定位稳定。** + + + + + + + +# HashMap核心方法源码分析 +## putVal() +```java +final V putVal(int hash, K key, V value, boolean onlyIfAbsent, + boolean evict) { + + // tab:HashMap 底层数组 + // p:当前桶的第一个节点 + // n:数组长度 + // i:key 对应的桶下标 + Node[] tab; Node p; int n, i; + + // 如果 table 还没初始化,或者长度为 0,则先 resize 初始化数组 + // new HashMap<>() 第一次 put 时,会在这里创建默认长度 16 的数组 + if ((tab = table) == null || (n = tab.length) == 0) + n = (tab = resize()).length; + + // 计算桶下标:(n - 1) & hash + // 因为 n 是 2 的幂,所以等价于 hash % n,但效率更高 + // 如果这个桶为空,直接放入新 Node + if ((p = tab[i = (n - 1) & hash]) == null) + tab[i] = newNode(hash, key, value, null); + + else { + // e:最终找到的旧节点;如果为 null,说明是新增 key + // k:临时保存已有节点的 key + Node e; K k; + + // 先检查桶中第一个节点是否就是目标 key + // 先比 hash,再比 key 引用或 equals + if (p.hash == hash && + ((k = p.key) == key || (key != null && key.equals(k)))) + e = p; + + // 如果桶已经是红黑树结构,走红黑树插入/查找逻辑 + else if (p instanceof TreeNode) + e = ((TreeNode)p).putTreeVal(this, tab, hash, key, value); + + else { + // 桶是普通链表,遍历链表 + for (int binCount = 0; ; ++binCount) { + + // 如果下一个节点为空,说明没有找到相同 key + // 把新节点追加到链表尾部 + if ((e = p.next) == null) { + p.next = newNode(hash, key, value, null); + + // 链表长度达到树化阈值 8 时,尝试树化 + // 注意:treeifyBin 内部还会判断 table 长度是否 >= 64 + // 如果小于 64,优先扩容,不会真正树化 + if (binCount >= TREEIFY_THRESHOLD - 1) + treeifyBin(tab, hash); + + break; + } + + // 找到 hash 和 key 都相同的旧节点,停止遍历 + if (e.hash == hash && + ((k = e.key) == key || (key != null && key.equals(k)))) + break; + + // 继续向后遍历链表 + p = e; + } + } + + // e != null 表示找到了旧 key,不是新增,而是更新 value + if (e != null) { + V oldValue = e.value; + + // onlyIfAbsent 为 false:直接覆盖旧值 + // onlyIfAbsent 为 true:只有旧值为 null 时才覆盖 + // put() 传 false,putIfAbsent() 传 true + if (!onlyIfAbsent || oldValue == null) + e.value = value; + + // LinkedHashMap 扩展点:访问节点后回调 + // HashMap 中是空实现 + afterNodeAccess(e); + + // 返回旧值 + return oldValue; + } + } + + // 结构性修改次数 +1 + // 用于 fail-fast,比如迭代时检测并发修改 + ++modCount; + + // 新增节点后 size +1 + // 如果 size 超过扩容阈值 threshold,则扩容 + if (++size > threshold) + resize(); + + // LinkedHashMap 扩展点:插入节点后回调 + // HashMap 中是空实现 + afterNodeInsertion(evict); + + // 新增 key 时返回 null + return null; +} +``` + +从源码中可以看到几个细节。 + +### 1)**JDK8 的 HashMap 链表插入用的是尾插法**。 +```java +if ((e = p.next) == null) { + p.next = newNode(hash, key, value, null); + ... + break; +} +``` + +```plain +table[i] -> A -> B +``` + +插入新元素之后 + +```plain +table[i] -> A -> B -> C +``` + +对比 JDK7,JDK7 HashMap 扩容迁移时使用头插法,可能在并发扩容下形成环链表,导致死循环。JDK8 改了扩容迁移逻辑,并且普通链表插入也是尾插,能保持链表相对顺序。 + +JDK8 HashMap 仍然不是线程安全的。尾插法解决不了所有并发问题。并发 put 仍可能出现数据覆盖、丢数据、size 不准、扩容状态异常等问题。 + + + +### 2)JDK8 HashMap线程不安全原因分析 +JDK8 HashMap 不安全,不是因为还会像 JDK7 那样容易成环,而是因为 put、size++、resize、table 发布、链表/红黑树修改都没有加锁或 CAS,多线程并发读写会发生覆盖、丢数据、计数错误和可见性问题。 + + + +> **桶为空时,多线程操作桶,会直接覆盖table[i]** +> + +```plain +if ((p = tab[i = (n - 1) & hash]) == null) + tab[i] = newNode(hash, key, value, null); +``` + +此处最核心原因是操作同一个桶位置,没有加锁,也没有进行CAS,线程不安全。 + + + +> **链表尾插时,p.next 可能互相覆盖** +> + +```java +if ((e = p.next) == null) { + p.next = newNode(hash, key, value, null); + ... + break; +} +``` + +多线程尾插法容易导致p.next正确结果被覆盖。 + + + +> **++size不是原子操作** +> + +```java +if (++size > threshold) + resize(); +``` + +此处++size不是原子操作,会导致最终size结果不准确。 + + + + + +### 3)JDK7 HashMap线程不安全原因分析 +JDK7 并发扩容时,头插法会反转链表,两个线程交叉修改同一批 Entry 的 next 指针,就可能把 A.next 指向 B,同时又把 B.next 指回 A,形成死循环。 + +```plain +T1 线程处理原始链表: + +A -> B -> null + + +T2 线程头插迁移后,把指针改成: + +B -> A -> null + + +T1 线程继续按旧进度迁移,但读到了 T2 线程改过的 B.next: + +B.next = A + + +最后成了循环链表,变成下图: + +A -> B +^ | +|____| +``` + +## getNode() +getNode()方法源码如下 + +```java +final Node getNode(int hash, Object key) { + // tab:底层数组 + // first:桶里的第一个节点 + // e:遍历链表时的当前节点 + // n:数组长度 + // k:临时保存节点 key + Node[] tab; Node first, e; int n; K k; + + // table 不为空、长度大于 0,并且目标桶不为空,才继续查找 + if ((tab = table) != null && (n = tab.length) > 0 && + (first = tab[(n - 1) & hash]) != null) { + + // 先检查桶里的第一个节点 + // 先比 hash,再比 key 引用或 equals + if (first.hash == hash && + ((k = first.key) == key || (key != null && key.equals(k)))) + return first; + + // 第一个节点不是目标 key,并且后面还有节点 + if ((e = first.next) != null) { + + // 如果桶已经树化,走红黑树查找 + if (first instanceof TreeNode) + return ((TreeNode)first).getTreeNode(hash, key); + + // 普通链表,依次向后遍历 + do { + // 找到 hash 和 key 都匹配的节点,直接返回 + if (e.hash == hash && + ((k = e.key) == key || (key != null && key.equals(k)))) + return e; + + // 继续访问下一个节点 + } while ((e = e.next) != null); + } + } + + // table 为空、桶为空,或者遍历完没找到 + return null; +} +``` + + 核心流程: + +1.table 为空,直接返回 null。 + +2.根据 hash 定位桶下标。 + +3.先查桶里的第一个节点。 + +4.如果是红黑树,走树查找。 + +5.否则遍历链表。 + +6.找不到返回 null。 + +## hash() +hash()方法是HashMap中的hash扰动函数,作用是:把 key 的 hashCode() 再处理一下,让高位信息也参与到低位计算,减少哈希冲突。 + +```java +static final int hash(Object key) { + int h; + return (key == null) ? 0 : (h = key.hashCode()) ^ (h >>> 16); +} +``` + + 最核心的是这一段:(h = key.hashCode()) ^ (h >>> 16) + +> **为什么要这么做?** +> + +在HashMap中计算桶下标都需要通过:(n - 1) & hash 来计算。又因为n是2的幂,所以这个计算主要以来的是hash的**低位**,高位一直没有利用到。HashMap初始容量为16,则n-1=15,15的二进制位:0000 1111,那么:(n - 1) & hash由于与操作的特性,这实际上只看hash的低4位。**这会导致:如果很多 key 的低位相同,即使高位不同,也会落到同一个桶里。** + +所以 HashMap 做了这个扰动:h ^ (h >>> 16)。把高 16 位右移到低 16 位,再和原 hash 异或,让高位信息参与低位计算。 + +举例: + +```java +>>>是无符号右移:整体向右移动,左边补 0,右边被移出去的低位丢弃。 + +原始 hash: + +高 16 位 低 16 位 +AAAA AAAA BBBB BBBB + +h >>> 16: + +0000 0000 AAAA AAAA + +异或后: + +AAAA AAAA (BBBB BBBB ^ AAAA AAAA) +``` + +## resize() +resize也是HashMap的核心方法之一,源码如下: + +```java +final Node[] resize() { + // 旧数组 + Node[] oldTab = table; + + // 旧容量,table 为空则为 0 + int oldCap = (oldTab == null) ? 0 : oldTab.length; + // 旧扩容阈值 + int oldThr = threshold; + // 新容量、新阈值 + int newCap, newThr = 0; + + // 情况一:旧数组已经存在,说明是正常扩容 + if (oldCap > 0) { + // 已经达到最大容量,不能再扩容 + if (oldCap >= MAXIMUM_CAPACITY) { + threshold = Integer.MAX_VALUE; + return oldTab; + } + // 容量扩大 2 倍 + // 阈值也扩大 2 倍 + else if ((newCap = oldCap << 1) < MAXIMUM_CAPACITY && + oldCap >= DEFAULT_INITIAL_CAPACITY) + newThr = oldThr << 1; + } + + // 情况二:数组还没创建,但 threshold > 0 + // 说明构造 HashMap 时指定了初始容量 + // 例如 new HashMap<>(32) + // 此时 threshold 暂时存的是 tableSizeFor(initialCapacity) + else if (oldThr > 0) + newCap = oldThr; + + // 情况三:无参构造 new HashMap<>() + // 第一次 put 时走这里,使用默认容量 16,默认阈值 12 + else { + newCap = DEFAULT_INITIAL_CAPACITY; + newThr = (int)(DEFAULT_LOAD_FACTOR * DEFAULT_INITIAL_CAPACITY); + } + + // 如果上面没有算出新阈值,则按 newCap * loadFactor 计算 + if (newThr == 0) { + float ft = (float)newCap * loadFactor; + newThr = (newCap < MAXIMUM_CAPACITY && ft < (float)MAXIMUM_CAPACITY ? + (int)ft : Integer.MAX_VALUE); + } + + // 更新扩容阈值 + threshold = newThr; + // 创建新数组 + @SuppressWarnings({"rawtypes","unchecked"}) + Node[] newTab = (Node[])new Node[newCap]; + // table 指向新数组 + table = newTab; + + // 如果旧数组不为空,需要迁移旧数据 + if (oldTab != null) { + + // 遍历旧数组每个桶 + for (int j = 0; j < oldCap; ++j) { + Node e; + // 如果当前桶不为空 + if ((e = oldTab[j]) != null) { + + // 旧桶置空,帮助 GC + oldTab[j] = null; + // 情况一:桶里只有一个节点,直接重新计算下标放入新数组 + if (e.next == null) + newTab[e.hash & (newCap - 1)] = e; + // 情况二:桶里是红黑树,走红黑树拆分逻辑 + else if (e instanceof TreeNode) + ((TreeNode)e).split(this, newTab, j, oldCap); + // 情况三:桶里是链表 + else { + // lo 链:扩容后仍然留在原下标 j + Node loHead = null, loTail = null; + // hi 链:扩容后移动到 j + oldCap + Node hiHead = null, hiTail = null; + Node next; + + // 遍历旧链表,把节点拆成 lo 和 hi 两条链 + do { + // 先保存下一个节点 + next = e.next; + // 判断扩容后位置是否不变 + if ((e.hash & oldCap) == 0) { + if (loTail == null) + loHead = e; + else + loTail.next = e; + loTail = e; + } + // 扩容后位置变为 原下标 + oldCap + else { + if (hiTail == null) + hiHead = e; + else + hiTail.next = e; + hiTail = e; + } + + } while ((e = next) != null); + + // lo 链放回原下标 j + if (loTail != null) { + loTail.next = null; + newTab[j] = loHead; + } + // hi 链放到新下标 j + oldCap + if (hiTail != null) { + hiTail.next = null; + newTab[j + oldCap] = hiHead; + } + } + } + } + } + + // 返回新数组 + return newTab; +} +``` + + + + + +# 总结 +HashMap中最核心的概念如下 + +```java +数组 + 链表 + 红黑树 +默认容量 16 +负载因子 0.75 +容量始终是 2 的幂 +链表长度 >= 8 且 table 容量 >= 64 时树化 +树节点过少时退化回链表 +允许 null key / null value +非线程安全 +``` + + + +JDK8和JDK7相比: + +```java +JDK7: +数组 + 链表 + +JDK8: +数组 + 链表 + 红黑树 +``` + From a1dedcd58aa5cc4c6e0fcfeb18f477f9f61ed30b Mon Sep 17 00:00:00 2001 From: coderbruis Date: Fri, 10 Jul 2026 20:33:38 +0800 Subject: [PATCH 24/37] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +- ...20\347\240\201\345\210\206\346\236\220.md" | 6 +- ...25\345\261\202\345\216\237\347\220\206.md" | 4 + ...73\350\276\221\345\210\206\346\236\220.md" | 136 ++++++++++++++---- 4 files changed, 120 insertions(+), 32 deletions(-) rename "note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" => "note/kafka/kafka Rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" (81%) diff --git a/README.md b/README.md index 5fffc49..b426ccb 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,10 @@ Kafka / RocketMQ 消息系统源码 ![kafka_architecture](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_architecture.png) Kafka Rebalance流程图 -![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) - ![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) +![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) + Kafka ISR / HW / LEO关系图 ![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) @@ -100,7 +100,7 @@ Kafka / RocketMQ 消息系统源码 - [深入分析 ThreadLocal](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%88%86%E6%9E%90ThreadLocal.md) - [深入学习 Java volatile 关键字](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Java%20volatile%E5%85%B3%E9%94%AE%E5%AD%97.md) - [深入学习 Thread 底层原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E5%AD%A6%E4%B9%A0Thread%E5%BA%95%E5%B1%82%E6%BA%90%E7%A0%81.md) -- [深入学习HashMap 底层源码与原理]() +- [深入学习HashMap 底层源码与原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/HashMap%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [开源项目里那些看不懂的位运算分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E5%BC%80%E6%BA%90%E9%A1%B9%E7%9B%AE%E9%87%8C%E9%82%A3%E4%BA%9B%E7%9C%8B%E4%B8%8D%E6%87%82%E7%9A%84%E4%BD%8D%E8%BF%90%E7%AE%97%E5%88%86%E6%9E%90.md) - [ThreadPoolExecutor 源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E6%9E%90ThreadPoolExecutor%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) diff --git "a/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" index 61386e6..a586cb2 100644 --- "a/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" +++ "b/note/JDK/HashMap\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -1,4 +1,8 @@ -当前HashMap版本:JDK1.8 ++ 当前HashMap版本:JDK1.8 ++ 转载请标明出处 + +# HashMap底层原理图 +![HashMap原理图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/HashMap.png) # HashMap的重要成员变量以及内部类 > **默认容量:DEFAULT_INITIAL_CAPACITY** diff --git "a/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" "b/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" index 02e5bd2..93a9b36 100644 --- "a/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" +++ "b/note/kafka/Kafka ISR \345\272\225\345\261\202\345\216\237\347\220\206.md" @@ -1,6 +1,10 @@ + 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:https://github.com/coderbruis/kafka source_code_analysis分支,底层原理持续更新) + 转载请标明出处 +# Kafka HW,ISR,LEO关系图 + +![Kafka_HW_ISR_LEO](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_HW_ISR_LEO.png) + # Kafka ISR是什么?解决什么问题? ## 是什么? Kafka ISR是In-Sync Replicas,意思是“与leader保持同步的副本集合”。在Kafka中会有leader副本和follower副本,下面举例: diff --git "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" "b/note/kafka/kafka Rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" similarity index 81% rename from "note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" rename to "note/kafka/kafka Rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" index 409bc93..bf3bde3 100644 --- "a/note/kafka/kafka rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" +++ "b/note/kafka/kafka Rebalance\346\240\270\345\277\203\351\200\273\350\276\221\345\210\206\346\236\220.md" @@ -1,6 +1,12 @@ + 当前分析版本是kafka最新版本(版本随时变化,最新分析代码请关注仓库:[https://github.com/coderbruis/kafka](https://github.com/coderbruis/kafka) **source_code_analysis分支**,底层原理持续更新) + 转载请标明出处 +# Kafka Rebalance流程图 + +![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) + +![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) + # Kafka Rebalance 核心流程 `KafkaConsumer.poll()` 是消费者触发 rebalance 的主要入口。 @@ -629,20 +635,47 @@ if (protocol == RebalanceProtocol.COOPERATIVE && # EAGER 和 COOPERATIVE 的差异 ## EAGER -EAGER rebalance 的特点是简单直接。 +EAGER核心特点:全量停、全量分、再恢复 + +### EAGER Rebalance流程 + +第一步:触发 Rebalance +1. 成员变化:join / leave / session timeout / max.poll.interval 超时等。 +2. 订阅或元数据变化:订阅 topic 变化、正则订阅匹配变化、partition 数变化等。 + +第二步:Coordinator 进入 Rebalance 状态 +1. Coordinator 将 group 状态切到 PreparingRebalance(可能不是第一次进入rebalance,所以这里状态已经是PreparingRebalance了,正常第一次是在JoinGroup进入Coordinator里,会将状态变更为PreparingRebalance。) +2. 现有成员会通过 heartbeat 或 poll 流程感知需要重新加入 group。 +3. 新成员或需要重分配的成员准备发送 JoinGroupRequest。 + +第三步:EAGER 全员撤销旧 assignment,然后 JoinGroup +1. 每个 consumer 在 JoinGroup 前执行 onJoinPrepare。 +2. EAGER 协议下 revoke 当前持有的所有 partitions。 +3. 调用 onPartitionsRevoked(allAssignedPartitions)。 +4. 清空本地 assignment,停止这些 partitions 的消费。全组成员进入消费暂停状态(STW)。 +5. 向 coordinator 发送 JoinGroupRequest,携带 subscription 和支持的 assignor。 + +第四步:JoinGroup 完成成员协商 +1. Coordinator 收集本轮成员的 JoinGroupRequest。 +2. 选择 leader consumer。 +3. 选择 assignment strategy / protocol,leader consumer生成新的assignment。 +4. 生成新的 generationId,同时将group状态变更为COMPLETING_REBALANCE。 +5. JoinGroupResponse 返回给成员。 +6. Leader 会拿到所有成员的 subscription metadata。 + +第五步:分配 + SyncGroup +1. Leader consumer 执行分配算法,比如 Range / Sticky。 +2. Leader 通过 SyncGroupRequest 把全组 assignment 发给 coordinator。 +3. Follower 也发送 SyncGroupRequest,但通常不带 assignment。 +4. Coordinator 保存本轮 assignment,group状态变更为STABLE。 +5. Coordinator 通过各自的 SyncGroupResponse 返回每个 consumer 自己的 assignment。 + +第六步:恢复消费 +1. Consumer 收到自己的 assignment。 +2. 更新本地 assignment。 +3. 调用 onPartitionsAssigned(newAssignedPartitions)。 +4. 从对应 offset 开始拉取消息,恢复消费。 -```plain -onJoinPrepare: - revoke all partitions - clear local assignment - -leader assign: - assign all partitions again - -onJoinComplete: - assign new partitions - trigger onPartitionsAssigned -``` 优点: @@ -655,23 +688,70 @@ onJoinComplete: + 即使某些 partition 仍然分给同一个 consumer,也会先 revoke 再 assign。 ## COOPERATIVE -COOPERATIVE rebalance 的特点是渐进迁移。 - -```plain -onJoinPrepare: - keep still-owned partitions - revoke only obviously invalid partitions +COOPERATIVE rebalance 的特点是渐进迁移,不会撤销所有分区导致全部分区STW。 + +### COOPERATIVE Rebalance流程 + +第一轮:标记迁移,旧 owner revoke(onwer表示持有parition的consumer) +1. 成员变化、订阅变化或元数据变化触发 rebalance。 +2. consumer 发送 JoinGroup 给 coordinator。 + JoinGroup metadata 里包含: + ○ 当前 subscription + ○ 当前本地已分配分区,也就是 ownedPartitions +3. coordinator 收集所有成员的 JoinGroup。 + coordinator 负责: + ○ 选 leader + ○ 选 assignor/protocol + ○ 把所有成员 subscription metadata 返回给 leader + coordinator收集完所有的JoinGroup请求之后,会进入新的generationId(递增)。 +4. leader consumer 执行 assignor。 + CooperativeStickyAssignor 会: + ○ 先计算目标 assignment + ○ 如果某个分区要从旧 owner 转给新 owner + ○ 但旧 owner 在本轮 JoinGroup 里仍上报该分区为 owned + ○ 那么本轮不会把该分区分给新 owner + ○ 同时旧 owner 的本轮 assignment 不再包含该分区 +5. leader Consumer 通过 SyncGroup 把全组 assignment 提交给 coordinator。其他consumer也会发送一个空的SyncGroup给coorinator +6. coordinator 保存 assignment,并通过 SyncGroupResponse 返回每个 consumer 自己的 assignment。 +7. consumer 处理 SyncGroupResponse。 + 本地计算: +```text + owned = 当前本地 assignment + assigned = 本轮收到的 assignment + + revoked = owned - assigned + added = assigned - owned +``` + +需要注意,第一轮主要是做撤销,但是也可能直接添加新的分区分配结果。因为如果这一轮有某个分区本来就没有旧onwer(旧的持有这个parition的consumer),这一轮就可以直接分配给新onwer。 +8. 如果 revoked 非空: + ○ 调用 onPartitionsRevoked(revoked) + ○ 调用 requestRejoin() + ○ 后续把本地 assignment 更新为 assigned +9. 如果 added 非空: + ○ 调用 onPartitionsAssigned(added) + 注意:正在从旧 owner 转移给新 owner 的分区,第一轮不会出现在新 owner 的 added 里。 + 第二轮:旧owner撤销分区,新owner获得第一轮撤销的分区 +1. 因为第一轮有 consumer 调用了 requestRejoin(),group 再次 rebalance。 +2. consumer 再次发送 JoinGroup。 + 此时旧 owner 的本地 assignment 已经更新,所以它上报的 ownedPartitions 不再包含刚刚 revoked 的分区。 + 被撤销的分区不会通过JoinGroup发送给Coordinator,revokeList和addLIst都是通过集合计算来得出的。 + coordinator收集完所有的JoinGroup请求之后,会进入新的generationId(递增)。 +3. coordinator 再次收集成员,选 leader,选协议。 + 然后将所有成员 subscription metadata 返回给 leader consumer。 +4. leader 再次执行 assignor。 + 这次 assignor 发现: + ○ 该分区已经没有旧 owner 上报 owned + ○ 可以安全分配给新 owner +5. leader 通过 SyncGroup 提交新的全组 assignment。 +6. coordinator 通过 SyncGroupResponse 返回各成员自己的 assignment。 +7. 新 owner 处理 assignment。 + 本地计算: + added = assigned - owned +8. 新 owner 对新增分区调用: + onPartitionsAssigned(added) -leader assign: - do not immediately reassign partitions still owned by others -onJoinComplete: - revoke partitions that need migration - request another rejoin if needed - -next rebalance: - assign released partitions to new owners -``` 优点: From 8ab30cd0f7d23e527f5d77fbf8c6b2d93fd17a4f Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sat, 11 Jul 2026 10:50:31 +0800 Subject: [PATCH 25/37] =?UTF-8?q?=E6=96=B0=E5=A2=9EAQS=E5=8E=9F=E7=90=86?= =?UTF-8?q?=E5=88=86=E6=9E=90=E6=96=87=E7=AB=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 + ...20\347\240\201\345\210\206\346\236\220.md" | 1060 +++++++++++++++++ 2 files changed, 1063 insertions(+) create mode 100644 "note/JDK/AQS\346\272\220\347\240\201\345\210\206\346\236\220.md" diff --git a/README.md b/README.md index b426ccb..158573c 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,9 @@ Kafka / RocketMQ 消息系统源码 HashMap原理图 ![HashMap原理图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/HashMap.png) +AQS架构图 +![AQS架构图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/AQS.png) +
diff --git "a/note/JDK/AQS\346\272\220\347\240\201\345\210\206\346\236\220.md" "b/note/JDK/AQS\346\272\220\347\240\201\345\210\206\346\236\220.md" new file mode 100644 index 0000000..8fb00bc --- /dev/null +++ "b/note/JDK/AQS\346\272\220\347\240\201\345\210\206\346\236\220.md" @@ -0,0 +1,1060 @@ +AQS版本:JDK 1.8 + +# AQS灵魂三问 +## 是什么? +AQS 全称是 AbstractQueuedSynchronizer,中文通常翻译为:**抽象队列同步器**。 + +AQS 本身不是一把可以直接使用的锁,而是一个用于构建锁和同步器的基础框架。 + +JDK 中很多并发工具都基于 AQS 实现: + +```java +ReentrantLock +ReentrantReadWriteLock +Semaphore +CountDownLatch +ThreadPoolExecutor.Worker +``` + +## 解决什么问题? +AQS 主要解决两类通用问题: + +1. 如何通过一个同步状态 state 判断资源是否可获取 +2. 获取失败后,如何让线程排队、阻塞,并在合适时机被唤醒 + + + +AQS 用于统一解决线程竞争、等待队列、阻塞与唤醒问题。 + +## 起什么作用? +AQS是 ReentrantLock、Semaphore 等同步器的基础框架。 + +AQS 将同步器拆成了两部分: + +1. 子类负责: + 1. 定义 state 的业务含义。 + 2. 定义获取资源和释放资源的规则。 +2. AQS负责: + 1. CAS 修改状态。 + 2. 维护等待队列。 + 3. 阻塞线程。 + 4. 唤醒线程。 + 5. 处理中断、超时和取消。 + +例如: + +1. ReentrantLock:state 表示锁的重入次数 +2. CountDownLatch:state 表示剩余计数 +3. Semaphore:state 表示剩余许可证数量 + +**** + +**AQS 的核心内容可以浓缩为:** + +1. 一个 volatile int state +2. 一个双向同步等待队列 +3. CAS 原子操作 +4. LockSupport.park/unpark 阻塞与唤醒 + +# AQS整体架构 +AQS 支持两种资源获取模式: + +1. 独占模式 Exclusive:同一时刻只允许一个线程成功获取资源。 +2. 共享模式 Shared:同一时刻允许多个线程成功获取资源。 + + + +典型应用: + +1. 独占模式:ReentrantLock、ReentrantReadWriteLock 写锁。 +2. 共享模式:Semaphore、CountDownLatch、ReentrantReadWriteLock 读锁。 + + + +两种模式共用同一个同步队列: + +```plain +head tail + | | + v v +[哨兵节点] <-> [独占节点 T1] <-> [共享节点 T2] <-> [独占节点 T3] +``` + + + +AQS 不理解“锁”“许可证”“计数器”等业务含义。它只负责: + +1. tryAcquire/tryAcquireShared 成功:线程继续执行。 +2. tryAcquire/tryAcquireShared 失败:线程进入队列并阻塞。 +3. tryRelease/tryReleaseShared 成功:唤醒后继线程。 + + + + +![](https://cdn.nlark.com/yuque/0/2026/png/12890164/1783689319993-5d064d1b-9d0e-4f6f-b208-5fbaedbc0952.png) + + + +# AQS核心结论 +AQS 本质上是一个用于构建锁和同步器的基础框架,核心由两部分组成:一个表示同步状态的 state,以及一个保存等待线程的 FIFO 双向队列。 + + + +线程获取资源时,首先通过 CAS 尝试修改 state: + ++ 获取成功:线程继续执行。 ++ 获取失败:线程封装成 Node 加入等待队列,并通过 LockSupport.park() 阻塞。 ++ 资源释放:修改 state,再通过 LockSupport.unpark() 唤醒后继节点重新竞争。 + + + +AQS 自身不定义资源如何获取和释放,而是通过模板方法交给子类实现: + ++ tryAcquire() ++ tryRelease() ++ tryAcquireShared() ++ tryReleaseShared() + + + +因此,AQS 的核心可以概括为: + ++ state 表示资源或锁状态。 ++ CAS 保证状态修改的原子性。 ++ 等待队列管理获取资源失败的线程。 ++ park/unpark 实现线程阻塞与唤醒。 ++ 模板方法定义具体同步规则。 ++ 支持独占模式和共享模式。 ++ ConditionObject 提供条件等待队列。 + + + +AQS 将“资源竞争、线程排队、阻塞与唤醒”统一封装,子类只需要实现资源的获取和释放规则。 + +# AQS 的重要成员变量和内部类 +## state:同步状态 +state 是 AQS 中最核心的变量: + +```java +private volatile int state; +``` + +AQS 使用一个 volatile int 保存同步状态,并通过以下方法读取和修改: + +```java +protected final int getState() +protected final void setState(int newState) +protected final boolean compareAndSetState(int expect, int update) +``` + +其中: + +```plain +getState():读取当前同步状态 +setState():直接设置同步状态 +compareAndSetState():通过 CAS 原子更新同步状态 +``` + +state 的含义由具体同步器决定。 + + + +ReentrantLock 中的 state表示 + ++ state = 0:锁未被任何线程持有 ++ state = 1:当前线程第一次获取锁 ++ state = 2:当前线程重入一次 ++ state = 3:当前线程重入两次 + + + +Semaphore 中的 state表示当前剩余许可证数量。 + +例如: + +```plain +初始许可证数量为 3 + +state = 3:还可以有 3 个线程获取 +state = 2:还可以有 2 个线程获取 +state = 0:没有剩余许可证 +``` + + + +CountDownLatch中的state表示尚未完成的计数。 + +例如: + +```plain +new CountDownLatch(3) + +初始 state = 3 +每次 countDown(),state 减 1 +state = 0 时,所有 await() 线程可以继续执行 +``` + +**** + +**总结:AQS 只提供一个线程安全的状态字段,不规定状态的业务含义。** + + + +## head 和 tail:同步队列首尾节点 +```java +private transient volatile Node head; +private transient volatile Node tail; +``` + +AQS 使用一个基于 CLH 思想改造的双向链表维护获取资源失败的线程。 + +```plain +head tail + | | + v v +[哨兵节点] <-> [节点 T1] <-> [节点 T2] <-> [节点 T3] +``` + +其中: + +```plain +head:队列头节点,通常是已经获取过资源的哨兵节点 + +tail:队列尾节点,新节点通过 CAS 追加到 tail +``` + +head和tail都使用 volatile 修饰,保证不同线程之间的可见性。 + + + +AQS 创建时不会立刻创建队列。只有第一次发生竞争、线程需要入队时,才会初始化一个空的哨兵节点: + +```plain +head == tail == new Node() +``` + +这样可以避免在从未发生竞争的同步器上浪费节点对象。 + +## Node:同步队列节点 +AQS 内部通过Node表示一个等待线程。 + +源码结构如下: + +```java +static final class Node { + ... + static final Node SHARED = new Node(); + static final Node EXCLUSIVE = null; + + static final int CANCELLED = 1; + static final int SIGNAL = -1; + static final int CONDITION = -2; + static final int PROPAGATE = -3; + + volatile int waitStatus; + // 指向同步队列中的前驱节点。 + volatile Node prev; + volatile Node next; + // 保存当前节点对应的等待线程。 + // 当节点需要被唤醒时,AQS 会执行:LockSupport.unpark() + volatile Thread thread; + Node nextWaiter; + ... +} +``` + +### prev +prev表示指向同步队列中的前驱节点。 + +```java +volatile Node prev; +``` + +指向同步队列中的前驱节点。 + +AQS 判断当前节点能否尝试获取资源时,最重要的条件是:**当前节点的前驱节点是否是 head。** + +只有队列中的第一个有效等待节点,才有资格再次调用tryAcquire()或tryAcquireShared()竞争资源。 + + + +AQS中判断源码如下: + +独占模式acquire: + +```java +final boolean acquireQueued(final Node node, int arg) { + ... + final Node p = node.predecessor(); + if (p == head && tryAcquire(arg)) { + ... + } + ... +} +``` + +共享模式acquire: + +```java +private void doAcquireShared(int arg) { + ... + final Node p = node.predecessor(); + if (p == head) { + ... + } + ... +} +``` + + + +**所以“当前节点的前驱节点是否是 head”是非常重要的一个条件。** + + + +### next(设计思想非常的细节) +指向同步队列中的后继节点。 + +```java +volatile Node next; +``` + +释放资源时,AQS 通常通过 head.next 找到需要唤醒的线程。 + +但next只是一个优化路径,不是绝对可靠的队列判断依据。 + +原因是节点入队过程如下: + +1. node.prev = oldTail +2. CAS 把 tail 从 oldTail 修改为 node +3. oldTail.next = node + +```java +// 入队操作,并返回当前node的前序节点 +private Node enq(final Node node) { + for (;;) { + Node t = tail; + if (t == null) { + if (compareAndSetHead(new Node())) + tail = head; + } else { + // 1. oldTail <- node + node.prev = t; + // 2. oldTail <- node(tail) 将node赋值给tail,表明当前node成为了tail + if (compareAndSetTail(t, node)) { + // oldTail -> node(tail) + t.next = node; + // 返回oldTail + return t; + } + } + } +} +``` + + + +在线程完成第 2 步、尚未完成第 3 步时: + +```plain +tail 已经指向新节点 +但 oldTail.next 仍然是 null +``` + +因此在部分场景下,如果判断到next=null,则AQS 会从tail沿着prev反向扫描,寻找有效后继节点。 + +```java +private void unparkSuccessor(Node node) { + ... + Node s = node.next; + if (s == null || s.waitStatus > 0) { + s = null; + // 通过next找不到要unpark的节点,则从tail开始向前遍历 + for (Node t = tail; t != null && t != node; t = t.prev) + if (t.waitStatus <= 0) + s = t; + } + ... +} +``` + + + +**总结:**因为有 next 指针,所以 release 时可以直接取 head,再通过 head.next 找到要唤醒的后继节点,通常是 O(1)。而如果没有next指针,则需要从tail尾结点向前通过prev遍历,时间复杂度为O(n)。所以在入队维护next指针的时候,因为: + +1)node.prev = t; + +2)compareAndSetTail(t, node) + +3)t.next = node; + +导致可能t.next会存在null的场景,所以通过next指针遍历会拿不到数据。所以如果next为null,就会退化成通过prev指针去获取目标节点。 + + + +### nextWaiter +nextWaiter 有两个用途。 + +#### 用途一:标识节点模式 +```java +nextWaiter == Node.SHARED:共享节点 +nextWaiter == Node.EXCLUSIVE:独占节点 +``` + +其中 Node.EXCLUSIVE 实际是 null。 + +#### 用途二:连接 Condition 条件队列 +Condition 条件队列是单向链表,节点之间通过 `nextWaiter` 连接: + +```plain +firstWaiter + | + v +[T1 CONDITION] -> [T2 CONDITION] -> [T3 CONDITION] + ^ + | + lastWaiter +``` + +所以: + +```plain +prev/next:用于 AQS 同步队列 +nextWaiter:用于 Condition 条件队列,或者标记共享模式 +``` + + + +### waitStatus 状态详解 +Node 中最难理解的字段是: + +```java +volatile int waitStatus; +``` + +它表示节点当前的等待状态。 + +JDK 8 中主要有以下几种状态: + ++ CANCELLED = 1 ++ SIGNAL = -1 ++ CONDITION = -2 ++ PROPAGATE = -3 ++ 默认状态 = 0 + +**** + +#### 0:默认状态 +新创建的同步队列节点,`waitStatus` 默认为 0。 + +表示节点当前没有特殊状态。 + + + +#### CANCELLED = 1:节点已取消 +```java +static final int CANCELLED = 1; +``` + +出现以下情况时,节点可能被取消: + ++ 线程等待超时 ++ 线程在可中断等待中被中断 ++ 获取资源过程中发生异常 + + + +取消后的节点不会再次参与资源竞争。 + +AQS 会在后续遍历中跳过 waitStatus > 0 的节点。 + +需要注意:CANCELLED 是唯一的正数状态。 + +因此源码中经常通过下面的方式判断节点是否已取消:waitStatus > 0 + + + +#### SIGNAL = -1:后继节点需要被唤醒 +```java +static final int SIGNAL = -1; +``` + +**这是同步队列中最重要的状态。** + + + +假设队列结构如下:pred -> node + +当 pred.waitStatus == SIGNAL 时,表示: + +```plain +node 准备阻塞。 +当 pred 释放资源、成为无效头节点或被取消时,pred 负责唤醒 node。 +``` + +一个非常容易混淆的点是: + +**SIGNAL 状态保存在前驱节点上,但它表达的是后继节点需要被唤醒。** + +例如: + +```plain +head(waitStatus = SIGNAL) -> T1 +``` + +表示 T1 可以安全阻塞,head 对应的资源持有者释放资源时,需要唤醒 T1。 + + + +为什么不把 SIGNAL 放在当前节点上? + +因为释放资源时,释放线程主要操作当前 `head`,由前驱节点记录唤醒责任,可以减少对后继节点状态的竞争修改。 + + + +#### CONDITION = -2:节点正在 Condition 条件队列中等待 +```java +static final int CONDITION = -2; +``` + +线程调用 Condition.await() 后,会进入 Condition 条件队列,此时节点状态为 CONDITION。 + +```plain +Condition 条件队列: + +[T1 CONDITION] -> [T2 CONDITION] -> [T3 CONDITION] +``` + + + +当其他线程调用 signal() 后,节点会: + +```plain +waitStatus:CONDITION -> 0 +从 Condition 条件队列转移到 AQS 同步队列 +``` + +只有重新进入同步队列并再次获取锁后,`await()` 才会返回。 + + + +#### PROPAGATE = -3:共享模式继续传播 +```java +static final int PROPAGATE = -3; +``` + +PROPAGATE 只用于共享模式。 + +它用于记录:即使当前释放动作没有直接找到需要唤醒的节点,后续共享节点仍需要继续检查并传播唤醒。 + + + +该状态主要用于解决共享模式下并发获取、释放交错时可能出现的传播遗漏问题。 + +可以先把它理解为:共享唤醒传播标记。 + + + +## waitStatus 的正负设计 +AQS 中对 `waitStatus` 的判断非常精简: + +```plain +waitStatus > 0:节点已取消,需要跳过 +waitStatus < 0:节点处于有效的信号、条件或传播状态 +waitStatus = 0:普通初始状态 +``` + +这种设计让很多分支只需要判断正负,而不必逐个比较状态常量。 + + + +# AQS 同步队列为什么是 CLH 变体 +AQS 的同步队列通常被称为 CLH 队列,但它并不是原始的 CLH 自旋锁队列。 + +原始 CLH 队列的核心思想是:每个线程关注自己的前驱节点状态。 + + + +AQS 保留了这个思想,但做了改造: + +```plain +原始 CLH:线程主要自旋等待 +AQS:线程尝试几次后,通过 LockSupport.park 阻塞 + +原始 CLH:通常只需要前驱引用 +AQS:增加 prev 和 next,便于取消、唤醒和队列维护 +``` + + + +AQS 节点主要观察前驱节点: + +```plain +前驱是 head:尝试获取资源 +前驱为 SIGNAL:当前线程可以安全 park +前驱已取消:跳过取消节点,重新连接有效前驱 +``` + +因此,AQS 同步队列可以理解为:基于 CLH 思想改造的 FIFO 双向阻塞队列 + + + +# AQS 留给子类实现的五个方法 +AQS 将排队和阻塞机制实现好,但资源获取规则需要子类定义。 + +核心扩展方法如下: + +```java +protected boolean tryAcquire(int arg) +protected boolean tryRelease(int arg) +protected int tryAcquireShared(int arg) +protected boolean tryReleaseShared(int arg) +protected boolean isHeldExclusively() +``` + +默认实现都会抛出UnsupportedOperationException,子类根据需要选择实现独占模式、共享模式或两者。 + + + +# AQS多线程竞争入队流程 +## 独占锁场景 +下面按**独占锁**场景流程,比如ReentrantLock.lock()。 + +**背景:当前有T0、T1、T2三个线程竞争锁,默认T0已获得锁。** + +### 1. T0 已经持有锁 +```plain +state = 1 +owner = T0 + +同步队列:空 +head = null +tail = null +``` + +**** + +### 2. T1 来竞争,tryAcquire 失败,封装成 Node 入队 +AQS 执行:addWaiter(Node.EXCLUSIVE) + + + +队列第一次初始化,会先创建哨兵 head: + +```plain +head(dummy) <-> T1 + ^ + tail +``` + +此时节点状态: + +```plain +head.waitStatus = 0 +T1.waitStatus = 0 +``` + +### 3. T1 入队后马上自旋再次竞争,失败,准备阻塞 +T1 在 acquireQueued 里判断: + +```plain +p == head && tryAcquire(arg) +``` + +此时 `p == head`,但锁还被 T0 持有,所以失败。 + +然后进入: + +```plain +shouldParkAfterFailedAcquire(head, T1) +``` + +T1 会把自己的前驱 head 设置成 **SIGNAL状态**: + +```plain +head(SIGNAL) <-> T1(0) + ^ + tail +``` + + + +然后T1发现前序节点是SIGNAL状态,然后T1调用park()进入阻塞状态。 + + + +### 4. T2 也来竞争,失败后入队 +```plain +head(SIGNAL) <-> T1(0) <-> T2(0) + ^ + tail +``` + +T2 的前驱是 T1。T2 获取失败后,马上自旋再次竞争锁,失败之后会把前驱 T1 设置成 `SIGNAL`: + +```plain +head(SIGNAL) <-> T1(SIGNAL) <-> T2(0) + ^ + tail +``` + +然后T2发现前序节点是SIGNAL状态,然后T2调用park()进入阻塞状态。 + +### 5. T0 释放锁,AQS 从 head 唤醒后继 +T0 调用: + +```plain +unlock() + -> release(1) + -> tryRelease(1) + -> unparkSuccessor(head) +``` + +队列: + +```plain +head(SIGNAL) <-> T1(SIGNAL) <-> T2(0) +``` + +AQS 找: + +```plain +head.next == T1 +``` + +然后: + +```plain +LockSupport.unpark(T1.thread) +``` + +结果: + +```plain +T1 被唤醒 +T2 仍阻塞 +``` + +### 6. T1 被唤醒后重新竞争锁,成功 +T1 醒来后回到循环: + +```plain +p == head && tryAcquire(arg) +``` + +此时锁已经空了,所以 T1 成功获取锁。 + +然后: + +```plain +setHead(T1) +``` + +队列变成: + +```plain +head = T1 +T1(thread=null, prev=null, SIGNAL) <-> T2(0) + ^ + tail +``` + +旧的 dummy head 会断开,帮助 GC: + +```plain +T1 成为新的 head +``` + +注意: + +```plain +T1 成为 head 后,它本身不再代表一个等待线程; +它代表当前成功获取过锁的节点。 +``` + +### 7. T1 释放锁,再唤醒 T2 +T1 执行: + +```plain +unlock() + -> release(1) + -> unparkSuccessor(head) +``` + +此时: + +```plain +head(T1, SIGNAL) <-> T2(0) +``` + +AQS 唤醒: + +```plain +head.next == T2 +``` + +T2 醒来,重新执行: + +```plain +p == head && tryAcquire(arg) +``` + +成功后: + +```plain +setHead(T2) +``` + +队列变成: + +```plain +head = T2 +tail = T2 +``` + +如果后面没有等待节点: + +```plain +T2.waitStatus 通常是 0 +``` + +# AQS核心方法 +## acquire() +当某个线程发起锁获取,比如调用ReentrantLock.lock()方法时,调用链会走到AbstractQueuedSynchronizer.acquire()方法,源码如下: + +```java +public final void acquire(int arg) { +if (!tryAcquire(arg) && + acquireQueued(addWaiter(Node.EXCLUSIVE), arg)) + selfInterrupt(); +} +``` + +## nonfairTryAcquire() +tryAcquire()逻辑在AQS子类里,当前就是在ReentrantLock中,核心逻辑: + +```java +final boolean nonfairTryAcquire(int acquires) { + final Thread current = Thread.currentThread(); + // 获取AQS状态 + int c = getState(); + // 没有线程持有这个共享状态 + if (c == 0) { + // CAS 变更state状态 + if (compareAndSetState(0, acquires)) { + // 标识当前线程持有state + setExclusiveOwnerThread(current); + return true; + } + } + // 如果是持有锁的线程 + else if (current == getExclusiveOwnerThread()) { + // state加一,重复持有锁 + int nextc = c + acquires; + if (nextc < 0) // overflow + throw new Error("Maximum lock count exceeded"); + setState(nextc); + return true; + } + return false; +} +``` + +nonfairTryAcquire()方法比较简单,主要就是通过CAS变更state状态,然后将持有锁的线程标识为owner,后续竞争锁则判断该线程是否持有,否则加锁失败返回false。 + + + +## addWaiter() +```java + private Node addWaiter(Node mode) { + Node node = new Node(Thread.currentThread(), mode); + // 尝试一次快速入队 + Node pred = tail; + // 如果pred=null,则表示队列还未初始化,快速入队逻辑不负责初始化 + if (pred != null) { + node.prev = pred; + // 并发竞争失败,会走到兜底入队逻辑 + if (compareAndSetTail(pred, node)) { + pred.next = node; + return node; + } + } + // 兜底入队逻辑 + enq(node); + return node; + } +``` + +addWaiter() 先用一次 CAS 快速入队;如果队列未初始化或发生并发竞争,就交给 enq() 自旋,确保当前线程节点最终进入 AQS 同步队列。 + + + +enq() 方法负责同步队列的延迟初始化,以及节点竞争入队。当队列未初始化时,通过 CAS 创建哨兵头节点;当队列已经初始化时,通过 CAS 将当前节点设置为新的尾节点。如果 CAS 失败,则不断自旋重试,直到节点成功入队。 + +```java + private Node enq(final Node node) { + for (;;) { + Node t = tail; + // 队列初始化 + if (t == null) { // Must initialize + if (compareAndSetHead(new Node())) + tail = head; + } else { + node.prev = t; + // CAS往队列尾部添加节点 + if (compareAndSetTail(t, node)) { + t.next = node; + // !返回的是入队节点的前序节点 + return t; + } + } + } + } +``` + +## acquireQueued() + + +```java +/** + * 已入队节点以独占模式获取同步状态。 + * + * @return 等待过程中是否发生过中断 + */ +final boolean acquireQueued(final Node node, int arg) { + boolean failed = true; + try { + boolean interrupted = false; + + // 自旋,直到成功获取同步状态 + for (;;) { + final Node p = node.predecessor(); + + // 只有头节点的直接后继才有资格尝试获取同步状态 + if (p == head && tryAcquire(arg)) { + // 获取成功,当前节点成为新的头节点 + setHead(node); + p.next = null; // 断开旧头节点,帮助 GC + failed = false; + return interrupted; + } + + // 获取失败,判断是否需要阻塞;被唤醒后检查中断状态 + if (shouldParkAfterFailedAcquire(p, node) && + parkAndCheckInterrupt()) { + interrupted = true; + } + } + } finally { + // 出现异常等获取失败的情况时,取消当前节点 + if (failed) { + cancelAcquire(node); + } + } +} +``` + +shouldParkAfterFailedAcquire()方法核心就是将入参node的前驱节点状态置为:SIGNAL,然后方法返回false,继续执行后面的parkAndCheckInterrupt(),将node节点置为阻塞状态。 + + + +这里就是对应着有新线程竞争锁失败之后,先加入队列,之后自旋尝试再次竞争锁,失败了则将前序节点置为SIGNAL,然后将自己阻塞等待被唤醒。 + + + +下面再分析下释放锁的流程,释放锁核心会调用AQS的release()方法。 + +```java +public final boolean release(int arg) { + // 子类负责更新锁状态,并判断锁是否已完全释放 + if (tryRelease(arg)) { + Node h = head; + + // 头节点存在且后继节点需要唤醒 + if (h != null && h.waitStatus != 0) + unparkSuccessor(h); + + return true; + } + + // 重入次数尚未归零,本次只减少持锁次数 + return false; +} +``` + + + +tryRelease()方法在ReentrantLock中,源码如下: + +```java +protected final boolean tryRelease(int releases) { + // 减少重入计数 + int c = getState() - releases; + + // 只有锁的持有线程才能释放锁 + if (Thread.currentThread() != getExclusiveOwnerThread()) + throw new IllegalMonitorStateException(); + + boolean free = false; + + // 重入计数归零,锁才算完全释放 + if (c == 0) { + free = true; + setExclusiveOwnerThread(null); + } + + // 更新剩余重入次数 + setState(c); + return free; +} +``` + + + +AQS的unparkSuccessor()源码如下: + +```java +/** + * 唤醒等待队列中有效的后继节点。 + */ +private void unparkSuccessor(Node node) { + int ws = node.waitStatus; + + // 清除当前节点的待唤醒标记 + if (ws < 0) + compareAndSetWaitStatus(node, ws, 0); + + Node s = node.next; + + // next 无效时,从队尾反向查找最靠前的有效等待节点 + if (s == null || s.waitStatus > 0) { + s = null; + for (Node t = tail; t != null && t != node; t = t.prev) { + // waitStatus > 0 表示节点已经取消 + if (t.waitStatus <= 0) + s = t; + } + } + + // 唤醒目标线程,让其重新参与锁竞争 + if (s != null) + LockSupport.unpark(s.thread); +} +``` + + + +释放锁流程总结: + +1. 当前线程调用 unlock(),最终进入 AQS 的 release(1)。 +2. release() 调用 tryRelease(1),减少 state 表示的重入次数。 +3. 如果当前线程不是锁的持有者,抛出 IllegalMonitorStateException。 +4. 如果 state 仍大于 0,说明当前线程还持有重入锁,不唤醒其他线程。 +5. 如果 state 减少到 0,清空锁的持有线程,表示锁已完全释放。 +6. release() 检查等待队列,通过 unparkSuccessor() 找到有效的后继节点。 +7. 调用 LockSupport.unpark() 唤醒对应线程,使其重新尝试获取锁。 + + + +需要注意:unpark() 只是让等待线程具备继续运行的条件,并不代表它立刻获得锁。线程被唤醒后,仍然需要参与锁竞争。 + From bfa742da0cf74f902fb2d8e3046181f4a109e416 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sat, 11 Jul 2026 10:52:38 +0800 Subject: [PATCH 26/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 158573c..05b1c54 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ Kafka / RocketMQ 消息系统源码 - [深入学习HashMap 底层源码与原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/HashMap%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [开源项目里那些看不懂的位运算分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E5%BC%80%E6%BA%90%E9%A1%B9%E7%9B%AE%E9%87%8C%E9%82%A3%E4%BA%9B%E7%9C%8B%E4%B8%8D%E6%87%82%E7%9A%84%E4%BD%8D%E8%BF%90%E7%AE%97%E5%88%86%E6%9E%90.md) - [ThreadPoolExecutor 源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/%E6%B7%B1%E5%85%A5%E8%A7%A3%E6%9E%90ThreadPoolExecutor%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) +- [AQS 源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/JDK/AQS%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) HashMap原理图 ![HashMap原理图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/HashMap.png) From 623f4101ac7c0817163af21aac5836e8655c35c0 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sun, 12 Jul 2026 08:54:05 +0800 Subject: [PATCH 27/37] =?UTF-8?q?=E9=87=8D=E5=B9=B3=E8=A1=A1=E4=B8=A5?= =?UTF-8?q?=E6=A0=BC=E5=8C=BA=E5=88=86=E4=B8=8D=E5=90=8C=E5=8D=8F=E8=AE=AE?= =?UTF-8?q?=E7=89=88=E6=9C=AC:=20Classic=20Consumer=20Group=20Protocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 05b1c54..41a5b54 100644 --- a/README.md +++ b/README.md @@ -73,13 +73,13 @@ Kafka / RocketMQ 消息系统源码 - [Kafka Broker 核心源码分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90.md) - [Kafka Broker 源码分析:生产者篇](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20broker%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%E2%80%94%E2%80%94%E7%94%9F%E4%BA%A7%E8%80%85%E7%AF%87.md) - [Kafka 消费者核心源码分析(一)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%E6%B6%88%E8%B4%B9%E8%80%85%E6%A0%B8%E5%BF%83%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90%EF%BC%88%E4%B8%80%EF%BC%89.md) -- [Kafka Rebalance 核心逻辑分析](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) +- [Kafka Rebalance 核心逻辑分析(Classic Consumer Group Protocol)](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/kafka%20Rebalance%E6%A0%B8%E5%BF%83%E9%80%BB%E8%BE%91%E5%88%86%E6%9E%90.md) - [Kafka ISR 原理](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/kafka/Kafka%20ISR%20%E5%BA%95%E5%B1%82%E5%8E%9F%E7%90%86.md) Kafka架构图 ![kafka_architecture](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_architecture.png) -Kafka Rebalance流程图 +Kafka Rebalance流程图(Classic Consumer Group Protocol 经典消费者组协议) ![Kafka EAGER Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_EAGER_rebalance.png) ![Kafka Cooperative Rebalance](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/kafka_COOPERATIVE_rebalance.png) From 39b2a718a77226f0962adf897f8bb366c03c531e Mon Sep 17 00:00:00 2001 From: CoderBruis <37364336+coderbruis@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:17:49 +0800 Subject: [PATCH 28/37] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=94=99=E5=88=AB?= =?UTF-8?q?=E5=AD=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...tor\345\272\225\345\261\202\345\216\237\347\220\206.md" | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git "a/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" "b/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" index ba9b143..8f55d40 100644 --- "a/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" +++ "b/note/JDK/\346\267\261\345\205\245\350\247\243\346\236\220ThreadPoolExecutor\345\272\225\345\261\202\345\216\237\347\220\206.md" @@ -11,7 +11,7 @@ private final AtomicInteger ctl = new AtomicInteger(ctlOf(RUNNING, 0)); -在不同操作系统下,Java 中的 Integer 变量都是32位,ThreadPoolExecutor 使用前3位(31~29)表示线程池状态,用后29位(28~0)表示活跃线程数。 +在不同操作系统下,Java 中的 Integer 变量都是32位,ThreadPoolExecutor 使用前3位(31 ~ 29)表示线程池状态,用后29位(28 ~ 0)表示活跃线程数。 ## COUNT_BITS COUNT_BITS的作用:用来划分 ctl 这个 int 变量的高低位边界。 @@ -22,7 +22,7 @@ private static final int COUNT_BITS = Integer.SIZE - 3; // 29 -int的最大位值是32位,32 - 3 = 29。通过这29来控制高3位存线程池状态**(注意这里不是线程,是线程池)**,低29位表示线程池中worker数量。通过位移来实现高效操作。 +int的最大位值是32位,32 - 3 = 29。通过这29来控制高3位存线程池状态(注意这里不是线程,是线程池),低29位表示线程池中worker数量。通过位移来实现高效操作。 ## CAPACITY CAPACITY的作用:用来表示 workerCount 线程数量部分的最大容量,同时也作为低 29 位的**掩码**。 @@ -40,11 +40,10 @@ private static final int CAPACITY = (1 << COUNT_BITS) - 1; 2)对左移结果 - 1 -(1 << COUNT_BITS) - 1:就是00100000 00000000 00000000 00000000 - 1,对于二进制,如果高位减1,会借位到对应位置-1。举例如下: +(1 << COUNT_BITS) - 1:就是00100000 00000000 00000000 00000000 - 1,对于二进制,如果高位减1,会借位到对应位置减1。举例如下: + 01000 - 1 = 01000 - 01111 = 00111 + 00100000 00000000 00000000 00000000 - 1 = 00100000 00000000 00000000 00000000 - 00111111 11111111 11111111 11111111 = 00011111 11111111 11111111 11111111 -+ 口诀:对于这种数:00100000 00000000 00000000 00000000,它是“某一位是 1,右边全是 0”。 ### 【扩展】掩码 掩码 Mask,就是一串二进制位,用来通过位运算“筛选”出你想要的部分。在ThreadPoolExecutor里: From 17d5b480db8c127ed08fb9227f7ab0d9277b247b Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 15 Jul 2026 20:25:47 +0800 Subject: [PATCH 29/37] =?UTF-8?q?=E6=96=B0=E5=A2=9ENetty=E6=9E=B6=E6=9E=84?= =?UTF-8?q?=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 41a5b54..5f8a663 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Kafka / RocketMQ 消息系统源码
Netty 底层源码解析 -- Netty 版本:4.1.43.Final +- Netty 版本:4.2 - [Netty 概念扫盲](https://github.com/coderbruis/JavaSourceCodeLearning/blob/master/note/Netty/Netty%E6%A6%82%E5%BF%B5%E6%89%AB%E7%9B%B2.md) - [二进制运算以及源码、反码以及补码学习](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/%E4%BA%8C%E8%BF%9B%E5%88%B6.md) - [Netty 源码包结构](https://github.com/coderbruis/JavaSourceLearning/blob/master/note/Netty/Netty%E6%BA%90%E7%A0%81%E5%8C%85%E7%BB%93%E6%9E%84.md) @@ -178,6 +178,12 @@ Kafka / RocketMQ 消息系统源码 Netty 实战课相关代码位于 `Spring-Netty` 模块下的 `com/bruis/learnnetty/im` 包。 +Netty架构图 +![Netty整体架构图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/Netty4.2.x.png) + +Netty主从Reactor架构图 +![Netty主从Reactor架构图](https://github.com/coderbruis/JavaSourceCodeLearning/releases/download/images-v1/Netty4.2.x.Reactor.png) +
From 59e16a86a1e61d46b1d8548d11e542f88bfdb79e Mon Sep 17 00:00:00 2001 From: coderbruis Date: Thu, 16 Jul 2026 13:58:31 +0800 Subject: [PATCH 30/37] =?UTF-8?q?=E6=9B=B4=E6=96=B0ThreadLocal=E5=86=85?= =?UTF-8?q?=E5=AD=98=E6=B3=84=E6=BC=8F=E4=BC=AA=E4=BB=A3=E7=A0=81=E5=88=86?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...245\345\210\206\346\236\220ThreadLocal.md" | 142 +++++++++++++++++- 1 file changed, 138 insertions(+), 4 deletions(-) diff --git "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" index ff6c108..44f8c01 100644 --- "a/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" +++ "b/note/JDK/\346\267\261\345\205\245\345\210\206\346\236\220ThreadLocal.md" @@ -150,7 +150,7 @@ Entry是ThreadLocalMap数组中的核心元素,它继承了WeakReference。核 Thread -> ThreadLocalMap -> Entry -> ThreadLocal ``` -在线程池场景下,线程可能长期存活。只要线程不结束,ThreadLocalMap 就还在,Entry 也还在,那么 ThreadLocal 对象就永远无法被 GC。即使业务代码已经不再持有这个 ThreadLocal 变量了,它也会被 Entry 强行引用住。这会导致:**ThreadLocal 对象无法回收,对应的 value 也无法回收。** +在线程池场景下,线程可能长期存活。只要线程不结束,ThreadLocalMap 就还在,Entry 也还在,那么 ThreadLocal 对象就永远无法被 GC。即使业务代码已经不再持有这个 ThreadLocal 变量了,它也会被 Entry 强行引用住。这会导致:**ThreadLocal 对象无法回收,对应的 value 也无法回收。** 这个后果就是线程长期持有已经没用的 ThreadLocal 和 value,导致内存释放不了,严重时内存泄漏、数据串用、甚至 OOM。 @@ -516,7 +516,9 @@ new Thread(() -> { }).start(); ``` - 而 InheritableThreadLocal 可以让子线程继承父线程的值: +```plain +而 InheritableThreadLocal 可以让子线程继承父线程的值: +``` ```java InheritableThreadLocal local = new InheritableThreadLocal<>(); @@ -576,7 +578,7 @@ private ThreadLocalMap(ThreadLocalMap parentMap) { -但是现在基本都没有直接通过new Thread()的方式创建线程了,基本都是通过线程池来管理线程。而在常规业务线程池里,InheritableThreadLocal 基本不适合作为上下文传递方案。它的继承时机是**“创建线程时”**,而线程池的线程通常早就创建好了,任务提交时不会重新继承父线程上下文。 +但是现在基本都没有直接通过new Thread()的方式创建线程了,基本都是通过线程池来管理线程。而在常规业务线程池里,InheritableThreadLocal 基本不适合作为上下文传递方案。它的继承时机是**“创建线程时”**,而线程池的线程通常早就创建好了,任务提交时不会重新继承父线程上下文。 线程池上下文传递方案,用的最多的就是阿里的TransmittableThreadLocal,简称 TTL。 @@ -620,7 +622,139 @@ class TtlRunnable implements Runnable { } ``` - + 总结:TransmittableThreadLocal在线程池里传值,是通过包装任务,在任务提交时捕获父线程的 TTL 快照,在工作线程执行前恢复这份快照,执行结束后再还原工作线程原上下文来实现的。 +# ThreadLocal内存泄漏代码分析 +下面是一段ThreadLocal内存泄漏的伪代码,通过这段伪代码加深ThreadLocal底层原理的理解。 + +```java +static final ThreadLocal USER_CONTEXT = new ThreadLocal<>(); + +void handleRequest(Request request) { + UserInfo userInfo = getUserInfo(request); + USER_CONTEXT.set(userInfo); + + doBusiness(); + + // 忘记执行 USER_CONTEXT.remove() +} +``` + +上述这段代码是用户登录之后,获取用户信息并存到ThreadLocal中,但是方法结束后并未执行:USER_CONTEXT.remove()移除ThreadLocal中的用户信息,这会造成内存泄漏,下面通过引用链来分析下内存泄漏的原因。 + +方法在USER_CONTEXT.set(userInfo)之后,引用关系如下图: + +```java +GC Roots +│ +├── ClassLoader +│ │ 强引用 +│ ▼ +│ Class对象 +│ │ 静态字段强引用 +│ ▼ +│ USER_CONTEXT +│ │ 强引用 +│ ▼ +│ ThreadLocal对象 ◀-------------------┐ +│ │ +└── 工作线程 Thread │ + │ │ + ├── 线程栈 │ + │ │ │ + │ ▼ │ + │ handleRequest()栈帧 │ + │ │ │ + │ ▼ │ + │ 局部变量 userInfo │ + │ │ 强引用 │ + │ ▼ │ + │ UserInfo对象 ◀──────────┐ │ + │ │ │ + └── threadLocals │ │ + │ 强引用 │ │ + ▼ │ │ + ThreadLocalMap │ │ + │ 强引用 │ │ + ▼ │ │ + Entry │ │ + ├── value强引用 ─────┘ │ + │ │ + └---- key弱引用 -----------┘ +``` + +这里有个细节,调用了USER_CONTEXT.set(userInfo)之后,ThreadLocal对象被USER_CONTEXT强引用引用,同时还被Entry这个弱引用给引用了。 + + + +当handleRequest()方法调用完之后,userInfo局部变量消失,此时引用关系图如下 + +```java +GC Roots +│ +├── ClassLoader +│ │ 强引用 +│ ▼ +│ Class对象 +│ │ 静态字段强引用 +│ ▼ +│ USER_CONTEXT +│ │ 强引用 +│ ▼ +│ ThreadLocal对象 ◀--------------------┐ +│ │ +└── 工作线程 Thread │ + │ │ + ├── 线程栈 │ + │ │ │ + │ └── handleRequest栈帧已消失 │ + │ │ + └── threadLocals │ + │ 强引用 │ + ▼ │ + ThreadLocalMap │ + │ 强引用 │ + ▼ │ + Entry │ + ├---- key弱引用 -----------┘ + │ + └── value强引用 + │ + ▼ + UserInfo对象 +``` + +此时发生变化的是,局部变量对应的引用已经断开。 + +```java +局部变量userInfo ──X──> UserInfo对象 +``` + + + +因此,即使局部变量消失,`UserInfo` 仍然无法被 GC 回收。只有执行 `USER_CONTEXT.remove()` 清理对应 Entry,才能断开这条引用链。 + + + +所以正确的写法如下 + +```java +private static final ThreadLocal USER_CONTEXT = + new ThreadLocal<>(); + +public void handleRequest(Request request) { + try { + UserInfo userInfo = getUserInfo(request); + USER_CONTEXT.set(userInfo); + + // 处理具体业务 + doBusiness(); + } finally { + // 无论正常结束还是发生异常,都必须清理 + USER_CONTEXT.remove(); + } +} +``` + From 7cf7a1b788b793999baa86c4734b2230f48aa3d5 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Fri, 17 Jul 2026 22:11:02 +0800 Subject: [PATCH 31/37] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=A9=BA=E6=96=87?= =?UTF-8?q?=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...15\350\257\225\346\234\272\345\210\266.md" | 0 ...23\345\255\230\346\234\272\345\210\266.md" | 0 ...20\347\240\201\346\220\255\345\273\272.md" | 52 ------------------- ...bo\351\235\242\350\257\225\351\242\230.md" | 22 -------- ...05\350\257\273\351\241\272\345\272\217.md" | 11 ---- 5 files changed, 85 deletions(-) delete mode 100644 "note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\344\272\224\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\351\207\215\350\257\225\346\234\272\345\210\266.md" delete mode 100644 "note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\345\233\233\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\347\274\223\345\255\230\346\234\272\345\210\266.md" delete mode 100644 "note/Dubbo/Dubbo\346\272\220\347\240\201\346\220\255\345\273\272.md" delete mode 100644 "note/Dubbo/Dubbo\351\235\242\350\257\225\351\242\230.md" delete mode 100644 "note/Dubbo/dubbo\345\272\225\345\261\202\345\216\237\347\220\206\351\230\205\350\257\273\351\241\272\345\272\217.md" diff --git "a/note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\344\272\224\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\351\207\215\350\257\225\346\234\272\345\210\266.md" "b/note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\344\272\224\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\351\207\215\350\257\225\346\234\272\345\210\266.md" deleted file mode 100644 index e69de29..0000000 diff --git "a/note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\345\233\233\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\347\274\223\345\255\230\346\234\272\345\210\266.md" "b/note/Dubbo/Dubbo\345\272\225\345\261\202\346\272\220\347\240\201\345\255\246\344\271\240\357\274\210\345\233\233\357\274\211\342\200\224\342\200\224 Dubbo\347\232\204\346\263\250\345\206\214\344\270\255\345\277\203\347\274\223\345\255\230\346\234\272\345\210\266.md" deleted file mode 100644 index e69de29..0000000 diff --git "a/note/Dubbo/Dubbo\346\272\220\347\240\201\346\220\255\345\273\272.md" "b/note/Dubbo/Dubbo\346\272\220\347\240\201\346\220\255\345\273\272.md" deleted file mode 100644 index 70b098d..0000000 --- "a/note/Dubbo/Dubbo\346\272\220\347\240\201\346\220\255\345\273\272.md" +++ /dev/null @@ -1,52 +0,0 @@ -## 前言 - -想要深入学习Dubbo,最好的方式就是阅读并调用Dubbo源码,接下来先来动手搭建一个Dubbo源码环境。 - -## 正文 - -### 1. 下载源码 - -步骤: - -1. 先从dubbo源码官网github中fork一份到自己的github仓库中。 - - ``` - git clone git@github.com:xxxxxxxx/dubbo.git - ``` - -2. 使用命令:git branch v2.7.8。 切换到分支2.7.8。 - - ``` - git checkout -b dubbo-2.7.8 dubbo-2.7.8 - ``` - -3. 导入方式(IDEA导入方式) - - 可以通过IDEA ——> File ——> Open ——> pom.xml ——> open as project - - 然后让IDEA下载相关的依赖,等下载完成即可。 - -4. mvn命令导入 - - ``` - mvn clean install -Dmaven.test.skip=true - ``` - - 然后执行下面的命令转换成 IDEA 项目: - - ``` - mvn idea:idea - ``` - - 如果执行报错了,则执行: - ``` - mvn idea:workspace - ``` - -### 2. 分支切换 - -本人Fork了官方Dubbo源码到本地仓库,并且新建了一个分支名为:SourceCode-of-Translation - -该分支主要用于进行源码注释,每个核心功能代码都有详细注释,欢迎大家Fork到本地,然后进行注释查看。 - -> 地址为:[SourceCode-of-Translation](https://github.com/coderbruis/dubbo) 下载到本地后,只需要切换到SourceCode-of-Translation分支即可。 diff --git "a/note/Dubbo/Dubbo\351\235\242\350\257\225\351\242\230.md" "b/note/Dubbo/Dubbo\351\235\242\350\257\225\351\242\230.md" deleted file mode 100644 index a8a3d5b..0000000 --- "a/note/Dubbo/Dubbo\351\235\242\350\257\225\351\242\230.md" +++ /dev/null @@ -1,22 +0,0 @@ -## 前言 - -Dubbo面试题 - -## 正文 - -RPC非常重要,很多人面试的时候都挂在了这个地方!你要是还不懂RPC是什么?他的基本原理是什么?你一定要把下边的内容记起来!好好研究一下!特别是文中给出的一张关于RPC的基本流程图,重点中的重点,Dubbo RPC的基本执行流程就是他,RPC框架的基本原理也是他,别说我没告诉你!看了下边的内容你要掌握的内容如下,当然还有很多: - -1. RPC的由来,是怎么一步一步演进出来的; -2. RPC的基本架构是什么; -3. RPC的基本实现原理, 重点; -4. REST和SOAP、PRC的区别; -5. 整个调用的过程经历了哪几部?和SpringMVC流程区别? - - -### 1. 为什么要有RPC - -随着互联网的发展,网站应用的规模不断扩大,常规的垂直应用架构已无法应对,分布式服务架构以及流动计算架构势在必行,亟需一个治理系统确保架构有条不紊的演进。 - -- [PRC原理图](https://github.com/coderbruis/JavaSourceLearning/blob/master/images/PRC/rpc.jpg) - - diff --git "a/note/Dubbo/dubbo\345\272\225\345\261\202\345\216\237\347\220\206\351\230\205\350\257\273\351\241\272\345\272\217.md" "b/note/Dubbo/dubbo\345\272\225\345\261\202\345\216\237\347\220\206\351\230\205\350\257\273\351\241\272\345\272\217.md" deleted file mode 100644 index 18929b7..0000000 --- "a/note/Dubbo/dubbo\345\272\225\345\261\202\345\216\237\347\220\206\351\230\205\350\257\273\351\241\272\345\272\217.md" +++ /dev/null @@ -1,11 +0,0 @@ -## 前言 - -在真正深入Dubbo底层源码前,先梳理好阅读路线,这样才能够事半功倍。 - -## 正文 - -### 1. 配置加载流程? - -[配置加载流程](http://dubbo.apache.org/zh-cn/docs/user/configuration/configuration-load-process.html) - -### 2. \ No newline at end of file From a7f505d5bbcb7f19b523626c40a2e71a7a129e67 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sat, 18 Jul 2026 22:03:26 +0800 Subject: [PATCH 32/37] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=9E=B6=E6=9E=84?= =?UTF-8?q?=E6=96=B9=E6=A1=88=E4=BC=AA=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture/README.md | 46 +++++ architecture/pom.xml | 78 ++++++++ .../arch/policy/ActiveSnapshotRegistry.java | 61 +++++++ .../arch/policy/DefaultSnapshotValidator.java | 15 ++ .../policy/FileSystemSnapshotDirectory.java | 45 +++++ .../java/com/arch/policy/MessagePosition.java | 20 +++ .../java/com/arch/policy/PolicyChange.java | 30 ++++ .../java/com/arch/policy/PolicyRecord.java | 27 +++ .../arch/policy/PolicySearchApplication.java | 15 ++ .../java/com/arch/policy/PolicySnapshot.java | 170 ++++++++++++++++++ .../arch/policy/PolicySnapshotService.java | 80 +++++++++ .../java/com/arch/policy/SnapshotBuilder.java | 56 ++++++ .../java/com/arch/policy/SnapshotPorts.java | 28 +++ .../aggregation/AsyncSearchCoordinator.java | 76 ++++++++ .../aggregation/LocalSearchWaiters.java | 32 ++++ .../arch/policy/aggregation/SearchState.java | 3 + .../policy/aggregation/SearchStateStore.java | 13 ++ .../aggregation/SupplierCallbackService.java | 38 ++++ .../aggregation/SupplierTaskDispatcher.java | 9 + .../com/arch/policy/api/CallbackResponse.java | 10 ++ .../arch/policy/api/PolicySearchRequest.java | 22 +++ .../arch/policy/api/PolicySearchResponse.java | 22 +++ .../policy/api/PolicySearchRpcService.java | 7 + .../policy/api/SupplierCallbackRequest.java | 25 +++ .../api/SupplierCallbackRpcService.java | 5 + .../application/PolicyIncrementalUpdater.java | 16 ++ .../application/PolicyStartupRunner.java | 19 ++ .../config/PolicySearchConfiguration.java | 109 +++++++++++ .../demo/DemoSupplierTaskDispatcher.java | 47 +++++ .../kafka/KafkaPolicyChangeListener.java | 45 +++++ .../policy/kafka/PolicyChangeMessage.java | 23 +++ .../policy/redis/RedisSearchStateStore.java | 84 +++++++++ .../redis/SearchFinishedSubscriber.java | 15 ++ .../policy/rpc/DubboPolicySearchService.java | 34 ++++ .../rpc/DubboSupplierCallbackService.java | 23 +++ .../src/main/resources/application.yml | 32 ++++ .../policy/AsyncSearchCoordinatorTest.java | 99 ++++++++++ .../arch/policy/SnapshotLifecycleTest.java | 119 ++++++++++++ 38 files changed, 1598 insertions(+) create mode 100644 architecture/README.md create mode 100644 architecture/pom.xml create mode 100644 architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java create mode 100644 architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java create mode 100644 architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java create mode 100644 architecture/src/main/java/com/arch/policy/MessagePosition.java create mode 100644 architecture/src/main/java/com/arch/policy/PolicyChange.java create mode 100644 architecture/src/main/java/com/arch/policy/PolicyRecord.java create mode 100644 architecture/src/main/java/com/arch/policy/PolicySearchApplication.java create mode 100644 architecture/src/main/java/com/arch/policy/PolicySnapshot.java create mode 100644 architecture/src/main/java/com/arch/policy/PolicySnapshotService.java create mode 100644 architecture/src/main/java/com/arch/policy/SnapshotBuilder.java create mode 100644 architecture/src/main/java/com/arch/policy/SnapshotPorts.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/SearchState.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java create mode 100644 architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java create mode 100644 architecture/src/main/java/com/arch/policy/api/CallbackResponse.java create mode 100644 architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java create mode 100644 architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java create mode 100644 architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java create mode 100644 architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java create mode 100644 architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java create mode 100644 architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java create mode 100644 architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java create mode 100644 architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java create mode 100644 architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java create mode 100644 architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java create mode 100644 architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java create mode 100644 architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java create mode 100644 architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java create mode 100644 architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java create mode 100644 architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java create mode 100644 architecture/src/main/resources/application.yml create mode 100644 architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java create mode 100644 architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java diff --git a/architecture/README.md b/architecture/README.md new file mode 100644 index 0000000..ed7e14c --- /dev/null +++ b/architecture/README.md @@ -0,0 +1,46 @@ +# Policy snapshot lifecycle + +This module implements version-isolated policy snapshots. Each snapshot owns a RocksDB detail +store, a bitmap index, and the incremental-message position at which both were built. + +## Integration flow + +1. Implement `FullPolicyLoader` to stream the full-data cut into `PolicySnapshot.upsert` and + return the cut's message position. +2. Implement `IncrementalReplayer` to apply upserts/deletes after that position. The builder + repeats replay until it reaches a stable latest position. +3. Supply `DefaultSnapshotValidator` (or a stricter domain validator), a + `FileSystemSnapshotDirectory`, and construct `SnapshotBuilder`. +4. Call `PolicySnapshotService.start(version)` during startup. Keep the application's readiness + probe bound to `service.isReady()`; failures remain unready and retry. +5. Call `refresh(newVersion)` at runtime. A failed candidate is discarded. A valid candidate is + atomically activated while the old snapshot continues serving existing leases. +6. Every search must use `try (SnapshotLease lease = registry.acquire())`. Releasing the last old + lease closes RocksDB and deletes that retired version's directory. + +The module intentionally leaves message-broker and full-data-source clients behind interfaces so +the snapshot consistency rules are independent of Kafka, HTTP, database, or framework choices. + +## Package layout + +- `api`: serializable Dubbo contract and request/response DTOs. +- `rpc`: Dubbo search provider and supplier callback provider. +- `aggregation`: Redis-backed fan-out/fan-in search coordination and local waiters. +- `redis`: atomic Lua state transitions and Pub/Sub early wake-up. +- `demo`: asynchronous downstream supplier simulation. +- `kafka`: JSON policy-change consumer and message DTO. +- `application`: startup and incremental-update use cases. +- root `policy` package: versioned RocksDB/Bitmap snapshot domain and lifecycle. + +Kafka messages use a globally monotonic `position` so duplicate/out-of-order delivery is ignored. +If the topic has multiple partitions, the producer must supply this global sequence; otherwise the +position model should be replaced with a per-partition offset map. + +## Async supplier search + +`asyncSearch` initializes pending suppliers, state and result TTL atomically in Redis, registers a +local waiter, double-checks Redis, then dispatches all supplier tasks. Supplier callbacks append +result chunks and remove a supplier from the pending set only on its final callback. The Lua script +sets `COMPLETED` and publishes `search-finished` when the last supplier finishes. Pub/Sub only wakes +the local waiter early; Redis remains the source of truth and is checked every 200 ms. Timeout is +also a Lua state transition and returns all partial results already recorded. diff --git a/architecture/pom.xml b/architecture/pom.xml new file mode 100644 index 0000000..4f7f396 --- /dev/null +++ b/architecture/pom.xml @@ -0,0 +1,78 @@ + + + 4.0.0 + + com.arch + architecture + 1.0-SNAPSHOT + + + 8 + 8 + UTF-8 + 5.10.2 + 2.7.18 + 3.2.15 + + + + + org.rocksdb + rocksdbjni + 8.11.3 + + + org.roaringbitmap + RoaringBitmap + 0.9.47 + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + org.springframework.boot + spring-boot-starter + ${spring-boot.version} + + + org.springframework.boot + spring-boot-starter-data-redis + ${spring-boot.version} + + + org.springframework.kafka + spring-kafka + 2.9.13 + + + com.fasterxml.jackson.core + jackson-databind + 2.13.5 + + + org.apache.dubbo + dubbo-spring-boot-starter + ${dubbo.version} + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot.version} + + + + diff --git a/architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java b/architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java new file mode 100644 index 0000000..f086e2c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java @@ -0,0 +1,61 @@ +package com.arch.policy; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** Atomic activation plus draining of queries that still hold the old version. */ +public final class ActiveSnapshotRegistry implements AutoCloseable { + private Entry active; + + public synchronized boolean isReady() { return active != null; } + + public synchronized SnapshotLease acquire() { + if (active == null) throw new IllegalStateException("policy snapshot is not ready"); + active.references++; + return new SnapshotLease(active); + } + + public synchronized void activate(PolicySnapshot snapshot) { + Entry previous = active; + active = new Entry(snapshot); + if (previous != null) retire(previous); + } + + @Override public synchronized void close() { + Entry previous = active; + active = null; + if (previous != null) retire(previous); + } + + private void release(Entry entry) { + synchronized (this) { + entry.references--; + closeWhenDrained(entry); + } + } + + private void retire(Entry entry) { + entry.retired = true; + closeWhenDrained(entry); + } + + private void closeWhenDrained(Entry entry) { + if (entry.retired && entry.references == 0) entry.snapshot.closeAndDelete(); + } + + private static final class Entry { + private final PolicySnapshot snapshot; + private int references; + private boolean retired; + private Entry(PolicySnapshot snapshot) { this.snapshot = snapshot; } + } + + public final class SnapshotLease implements AutoCloseable { + private final Entry entry; + private final AtomicBoolean released = new AtomicBoolean(); + private SnapshotLease(Entry entry) { this.entry = entry; } + public PolicySnapshot snapshot() { return entry.snapshot; } + @Override public void close() { + if (released.compareAndSet(false, true)) release(entry); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java b/architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java new file mode 100644 index 0000000..2c8c848 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java @@ -0,0 +1,15 @@ +package com.arch.policy; + +import static com.arch.policy.SnapshotPorts.SnapshotValidator; + +/** Baseline invariants; domain-specific checks can be supplied through SnapshotValidator. */ +public final class DefaultSnapshotValidator implements SnapshotValidator { + @Override public void validate(PolicySnapshot candidate, MessagePosition expectedPosition) throws Exception { + if (!candidate.getPosition().equals(expectedPosition)) { + throw new IllegalStateException("message position mismatch"); + } + if (candidate.policyCount() != candidate.indexedPolicyCount()) { + throw new IllegalStateException("RocksDB and bitmap policy counts differ"); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java b/architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java new file mode 100644 index 0000000..64c0bb1 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java @@ -0,0 +1,45 @@ +package com.arch.policy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +import static com.arch.policy.SnapshotPorts.SnapshotDirectory; + +/** Keeps every version in an isolated directory and removes abandoned candidates. */ +public final class FileSystemSnapshotDirectory implements SnapshotDirectory { + private final Path root; + + public FileSystemSnapshotDirectory(Path root) { this.root = root; } + + @Override public Path create(String version) throws IOException { + Path directory = root.resolve(safeVersion(version)); + delete(directory); + return Files.createDirectories(directory); + } + + @Override public void delete(Path directory) throws IOException { + if (!Files.exists(directory)) return; + try (Stream paths = Files.walk(directory)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> { + try { Files.deleteIfExists(path); } + catch (IOException failure) { throw new DeleteFailure(failure); } + }); + } catch (DeleteFailure failure) { + throw (IOException) failure.getCause(); + } + } + + private static String safeVersion(String version) { + if (version == null || !version.matches("[A-Za-z0-9._-]+")) { + throw new IllegalArgumentException("invalid snapshot version: " + version); + } + return version; + } + + private static final class DeleteFailure extends RuntimeException { + private DeleteFailure(IOException cause) { super(cause); } + } +} diff --git a/architecture/src/main/java/com/arch/policy/MessagePosition.java b/architecture/src/main/java/com/arch/policy/MessagePosition.java new file mode 100644 index 0000000..50794e1 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/MessagePosition.java @@ -0,0 +1,20 @@ +package com.arch.policy; + +public final class MessagePosition implements Comparable { + public static final MessagePosition BEGINNING = new MessagePosition(0); + private final long value; + + public MessagePosition(long value) { + if (value < 0) throw new IllegalArgumentException("position must be non-negative"); + this.value = value; + } + + public long getValue() { return value; } + + @Override public int compareTo(MessagePosition other) { return Long.compare(value, other.value); } + @Override public boolean equals(Object other) { + return other instanceof MessagePosition && value == ((MessagePosition) other).value; + } + @Override public int hashCode() { return Long.valueOf(value).hashCode(); } + @Override public String toString() { return Long.toString(value); } +} diff --git a/architecture/src/main/java/com/arch/policy/PolicyChange.java b/architecture/src/main/java/com/arch/policy/PolicyChange.java new file mode 100644 index 0000000..5dfcecc --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/PolicyChange.java @@ -0,0 +1,30 @@ +package com.arch.policy; + +public final class PolicyChange { + public enum Type { UPSERT, DELETE } + + private final Type type; + private final int policyId; + private final PolicyRecord policy; + private final MessagePosition position; + + private PolicyChange(Type type, int policyId, PolicyRecord policy, MessagePosition position) { + this.type = type; + this.policyId = policyId; + this.policy = policy; + this.position = position; + } + + public static PolicyChange upsert(PolicyRecord policy, MessagePosition position) { + return new PolicyChange(Type.UPSERT, policy.getId(), policy, position); + } + + public static PolicyChange delete(int policyId, MessagePosition position) { + return new PolicyChange(Type.DELETE, policyId, null, position); + } + + public Type getType() { return type; } + public int getPolicyId() { return policyId; } + public PolicyRecord getPolicy() { return policy; } + public MessagePosition getPosition() { return position; } +} diff --git a/architecture/src/main/java/com/arch/policy/PolicyRecord.java b/architecture/src/main/java/com/arch/policy/PolicyRecord.java new file mode 100644 index 0000000..42371e3 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/PolicyRecord.java @@ -0,0 +1,27 @@ +package com.arch.policy; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +public final class PolicyRecord { + private final int id; + private final byte[] detail; + private final Set indexTerms; + + public PolicyRecord(int id, byte[] detail, Set indexTerms) { + if (id < 0) { + throw new IllegalArgumentException("policy id must be non-negative"); + } + this.id = id; + this.detail = Arrays.copyOf(detail, detail.length); + this.indexTerms = Collections.unmodifiableSet(new HashSet(indexTerms)); + } + + public int getId() { return id; } + + public byte[] getDetail() { return Arrays.copyOf(detail, detail.length); } + + public Set getIndexTerms() { return indexTerms; } +} diff --git a/architecture/src/main/java/com/arch/policy/PolicySearchApplication.java b/architecture/src/main/java/com/arch/policy/PolicySearchApplication.java new file mode 100644 index 0000000..9bdfd8b --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/PolicySearchApplication.java @@ -0,0 +1,15 @@ +package com.arch.policy; + +import org.apache.dubbo.config.spring.context.annotation.EnableDubbo; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.kafka.annotation.EnableKafka; + +@EnableKafka +@EnableDubbo +@SpringBootApplication +public class PolicySearchApplication { + public static void main(String[] args) { + SpringApplication.run(PolicySearchApplication.class, args); + } +} diff --git a/architecture/src/main/java/com/arch/policy/PolicySnapshot.java b/architecture/src/main/java/com/arch/policy/PolicySnapshot.java new file mode 100644 index 0000000..d4e7e53 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/PolicySnapshot.java @@ -0,0 +1,170 @@ +package com.arch.policy; + +import org.roaringbitmap.RoaringBitmap; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +/** A version-isolated RocksDB detail store and its matching bitmap index. */ +public final class PolicySnapshot implements AutoCloseable { + static { RocksDB.loadLibrary(); } + + private final String version; + private final Path directory; + private final Options options; + private final RocksDB database; + private final Map bitmapIndex = new HashMap(); + private final RoaringBitmap allPolicyIds = new RoaringBitmap(); + private final Map> termsByPolicy = new HashMap>(); + private MessagePosition position = MessagePosition.BEGINNING; + private boolean closed; + + public PolicySnapshot(String version, Path directory) throws RocksDBException { + this.version = version; + this.directory = directory; + this.options = new Options().setCreateIfMissing(true); + this.database = RocksDB.open(options, directory.toString()); + } + + public synchronized void upsert(PolicyRecord policy) throws RocksDBException { + ensureOpen(); + removeFromIndex(policy.getId()); + database.put(key(policy.getId()), policy.getDetail()); + allPolicyIds.add(policy.getId()); + termsByPolicy.put(policy.getId(), policy.getIndexTerms()); + for (String term : policy.getIndexTerms()) { + RoaringBitmap bitmap = bitmapIndex.get(term); + if (bitmap == null) { + bitmap = new RoaringBitmap(); + bitmapIndex.put(term, bitmap); + } + bitmap.add(policy.getId()); + } + } + + public synchronized void delete(int policyId) throws RocksDBException { + ensureOpen(); + database.delete(key(policyId)); + allPolicyIds.remove(policyId); + removeFromIndex(policyId); + } + + public synchronized byte[] findDetail(int policyId) throws RocksDBException { + ensureOpen(); + return database.get(key(policyId)); + } + + public synchronized RoaringBitmap findPolicyIds(String term) { + ensureOpen(); + RoaringBitmap bitmap = bitmapIndex.get(term); + return bitmap == null ? new RoaringBitmap() : bitmap.clone(); + } + + public synchronized RoaringBitmap allPolicyIds() { + ensureOpen(); + return allPolicyIds.clone(); + } + + /** Applies an ordered event and advances its position in the same monitor. */ + public synchronized boolean apply(PolicyChange change) throws RocksDBException { + ensureOpen(); + if (change.getPosition().compareTo(position) <= 0) return false; + if (change.getType() == PolicyChange.Type.DELETE) delete(change.getPolicyId()); + else upsert(change.getPolicy()); + advanceTo(change.getPosition()); + return true; + } + + public synchronized long policyCount() throws RocksDBException { + ensureOpen(); + long count = 0; + try (RocksIterator iterator = database.newIterator()) { + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) count++; + iterator.status(); + } + return count; + } + + public synchronized Map bitmapCardinalities() { + ensureOpen(); + Map result = new HashMap(); + for (Map.Entry entry : bitmapIndex.entrySet()) { + result.put(entry.getKey(), entry.getValue().getCardinality()); + } + return Collections.unmodifiableMap(result); + } + + public synchronized int indexedPolicyCount() { + ensureOpen(); + RoaringBitmap all = new RoaringBitmap(); + for (RoaringBitmap bitmap : bitmapIndex.values()) all.or(bitmap); + return all.getCardinality(); + } + + public String getVersion() { return version; } + public Path getDirectory() { return directory; } + public synchronized MessagePosition getPosition() { return position; } + public synchronized void advanceTo(MessagePosition newPosition) { + ensureOpen(); + if (newPosition.compareTo(position) < 0) throw new IllegalArgumentException("position cannot move backwards"); + position = newPosition; + } + + @Override public synchronized void close() { + if (closed) return; + closed = true; + database.close(); + options.close(); + bitmapIndex.clear(); + allPolicyIds.clear(); + termsByPolicy.clear(); + } + + public void closeAndDelete() { + close(); + try { + if (!Files.exists(directory)) return; + try (Stream paths = Files.walk(directory)) { + paths.sorted(java.util.Comparator.reverseOrder()).forEach(path -> { + try { Files.deleteIfExists(path); } + catch (IOException failure) { throw new DeleteFailure(failure); } + }); + } + } catch (IOException failure) { + throw new IllegalStateException("cannot delete snapshot " + directory, failure); + } catch (DeleteFailure failure) { + throw new IllegalStateException("cannot delete snapshot " + directory, failure.getCause()); + } + } + + private static final class DeleteFailure extends RuntimeException { + private DeleteFailure(IOException cause) { super(cause); } + } + + private void removeFromIndex(int policyId) { + Set oldTerms = termsByPolicy.remove(policyId); + if (oldTerms == null) return; + for (String term : oldTerms) { + RoaringBitmap bitmap = bitmapIndex.get(term); + bitmap.remove(policyId); + if (bitmap.isEmpty()) bitmapIndex.remove(term); + } + } + + private void ensureOpen() { + if (closed) throw new IllegalStateException("snapshot is closed: " + version); + } + + private static byte[] key(int id) { return ByteBuffer.allocate(4).putInt(id).array(); } +} diff --git a/architecture/src/main/java/com/arch/policy/PolicySnapshotService.java b/architecture/src/main/java/com/arch/policy/PolicySnapshotService.java new file mode 100644 index 0000000..d90fac8 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/PolicySnapshotService.java @@ -0,0 +1,80 @@ +package com.arch.policy; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +/** Coordinates startup retry and side-by-side runtime refresh. */ +public final class PolicySnapshotService implements AutoCloseable { + private final SnapshotBuilder builder; + private final ActiveSnapshotRegistry registry; + private final ScheduledExecutorService executor; + private final long retryDelayMillis; + private final AtomicBoolean building = new AtomicBoolean(); + private volatile boolean stopped; + + public PolicySnapshotService(SnapshotBuilder builder, ActiveSnapshotRegistry registry, + ScheduledExecutorService executor, long retryDelayMillis) { + this.builder = builder; + this.registry = registry; + this.executor = executor; + this.retryDelayMillis = retryDelayMillis; + } + + /** Keeps readiness false and retries until the first valid snapshot is activated. */ + public void start(String version) { submitBuild(version, true); } + + /** Returns false when another candidate is already being built. */ + public boolean refresh(String version) { return submitBuild(version, false); } + + public boolean isReady() { return registry.isReady(); } + public boolean isBuilding() { return building.get(); } + + /** Used by application startup: Spring startup does not finish before a snapshot is ready. */ + public void initializeBlocking(String version) throws InterruptedException { + if (!building.compareAndSet(false, true)) throw new IllegalStateException("snapshot build already running"); + try { + while (!stopped && !registry.isReady()) { + try { + registry.activate(builder.build(version)); + } catch (Exception failure) { + Thread.sleep(retryDelayMillis); + } + } + } finally { + building.set(false); + } + } + + private boolean submitBuild(final String version, final boolean retryOnFailure) { + if (stopped || !building.compareAndSet(false, true)) return false; + executor.execute(new Runnable() { + @Override public void run() { buildAndActivate(version, retryOnFailure); } + }); + return true; + } + + private void buildAndActivate(final String version, final boolean retryOnFailure) { + try { + PolicySnapshot candidate = builder.build(version); + if (stopped) candidate.closeAndDelete(); + else registry.activate(candidate); + } catch (Exception ignored) { + if (retryOnFailure && !stopped) { + executor.schedule(new Runnable() { + @Override public void run() { buildAndActivate(version, true); } + }, retryDelayMillis, TimeUnit.MILLISECONDS); + return; + } + } finally { + // Startup retry owns the build slot until it succeeds or the service stops. + if (!retryOnFailure || registry.isReady() || stopped) building.set(false); + } + } + + @Override public void close() { + stopped = true; + registry.close(); + executor.shutdownNow(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/SnapshotBuilder.java b/architecture/src/main/java/com/arch/policy/SnapshotBuilder.java new file mode 100644 index 0000000..fbd2ccf --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/SnapshotBuilder.java @@ -0,0 +1,56 @@ +package com.arch.policy; + +import org.rocksdb.RocksDBException; + +import java.nio.file.Path; + +import static com.arch.policy.SnapshotPorts.*; + +/** Builds a candidate without exposing it to queries. */ +public final class SnapshotBuilder { + private final FullPolicyLoader fullLoader; + private final IncrementalReplayer replayer; + private final SnapshotValidator validator; + private final SnapshotDirectory directories; + + public SnapshotBuilder(FullPolicyLoader fullLoader, IncrementalReplayer replayer, + SnapshotValidator validator, SnapshotDirectory directories) { + this.fullLoader = fullLoader; + this.replayer = replayer; + this.validator = validator; + this.directories = directories; + } + + public PolicySnapshot build(String version) throws Exception { + Path directory = directories.create(version); + PolicySnapshot candidate = null; + try { + candidate = open(version, directory); + MessagePosition position = fullLoader.loadInto(candidate); + candidate.advanceTo(position); + catchUp(candidate); + validator.validate(candidate, candidate.getPosition()); + return candidate; + } catch (Exception failure) { + if (candidate != null) candidate.close(); + directories.delete(directory); + throw failure; + } + } + + private void catchUp(PolicySnapshot candidate) throws Exception { + while (true) { + MessagePosition target = replayer.latestPosition(); + if (candidate.getPosition().compareTo(target) >= 0) return; + MessagePosition replayed = replayer.replayInto(candidate, candidate.getPosition(), target); + if (replayed.compareTo(candidate.getPosition()) <= 0 || replayed.compareTo(target) > 0) { + throw new IllegalStateException("incremental replay made invalid progress"); + } + candidate.advanceTo(replayed); + } + } + + private PolicySnapshot open(String version, Path directory) throws RocksDBException { + return new PolicySnapshot(version, directory); + } +} diff --git a/architecture/src/main/java/com/arch/policy/SnapshotPorts.java b/architecture/src/main/java/com/arch/policy/SnapshotPorts.java new file mode 100644 index 0000000..9e3c4d4 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/SnapshotPorts.java @@ -0,0 +1,28 @@ +package com.arch.policy; + +import java.nio.file.Path; + +public final class SnapshotPorts { + private SnapshotPorts() {} + + public interface FullPolicyLoader { + /** Returns the message position corresponding to the full-data cut. */ + MessagePosition loadInto(PolicySnapshot target) throws Exception; + } + + public interface IncrementalReplayer { + /** Replays (position, latest] and returns the actual replayed position. */ + MessagePosition replayInto(PolicySnapshot target, MessagePosition position, + MessagePosition latest) throws Exception; + MessagePosition latestPosition() throws Exception; + } + + public interface SnapshotValidator { + void validate(PolicySnapshot candidate, MessagePosition expectedPosition) throws Exception; + } + + public interface SnapshotDirectory { + Path create(String version) throws Exception; + void delete(Path directory) throws Exception; + } +} diff --git a/architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java b/architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java new file mode 100644 index 0000000..a58d313 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java @@ -0,0 +1,76 @@ +package com.arch.policy.aggregation; + +import com.arch.policy.api.PolicySearchRequest; +import com.arch.policy.api.PolicySearchResponse; +import com.arch.policy.api.SupplierCallbackRequest; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + +public final class AsyncSearchCoordinator { + private static final long MAX_WAIT_SLICE_MILLIS = 200; + private final SearchStateStore stateStore; + private final SupplierTaskDispatcher dispatcher; + private final LocalSearchWaiters localWaiters; + private final ObjectMapper objectMapper; + private final long redisTtlSeconds; + + public AsyncSearchCoordinator(SearchStateStore stateStore, SupplierTaskDispatcher dispatcher, + LocalSearchWaiters localWaiters, ObjectMapper objectMapper, + long redisTtlSeconds) { + this.stateStore = stateStore; + this.dispatcher = dispatcher; + this.localWaiters = localWaiters; + this.objectMapper = objectMapper; + this.redisTtlSeconds = redisTtlSeconds; + } + + public PolicySearchResponse search(PolicySearchRequest request) throws Exception { + validate(request); + String searchKey = UUID.randomUUID().toString(); + Set suppliers = new LinkedHashSet(request.getSupplierIds()); + stateStore.initialize(searchKey, suppliers, redisTtlSeconds); + LocalSearchWaiters.Waiter waiter = localWaiters.register(searchKey); + try { + if (stateStore.getState(searchKey) == SearchState.COMPLETED) return aggregate(searchKey); + dispatcher.dispatch(searchKey, suppliers, request); + return awaitResults(searchKey, request.getTotalTimeoutMillis(), waiter); + } finally { + localWaiters.remove(searchKey, waiter); + } + } + + private PolicySearchResponse awaitResults(String searchKey, long timeoutMillis, + LocalSearchWaiters.Waiter waiter) throws Exception { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (true) { + SearchState state = stateStore.getState(searchKey); + if (state == SearchState.COMPLETED) return aggregate(searchKey); + long remaining = deadline - System.currentTimeMillis(); + if (state == SearchState.TIMED_OUT || remaining <= 0) { + stateStore.markTimedOut(searchKey, redisTtlSeconds); + return aggregate(searchKey); + } + waiter.await(Math.min(remaining, MAX_WAIT_SLICE_MILLIS)); + } + } + + private PolicySearchResponse aggregate(String searchKey) throws Exception { + List results = new ArrayList(); + for (String payload : stateStore.getResultPayloads(searchKey)) { + results.add(objectMapper.readValue(payload, SupplierCallbackRequest.class)); + } + SearchState state = stateStore.getState(searchKey); + return new PolicySearchResponse(searchKey, state.name(), results); + } + + private static void validate(PolicySearchRequest request) { + if (request == null) throw new IllegalArgumentException("request is required"); + if (request.getTotalTimeoutMillis() <= 0) throw new IllegalArgumentException("timeout must be positive"); + if (request.getSupplierIds().isEmpty()) throw new IllegalArgumentException("supplierIds is required"); + } +} diff --git a/architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java b/architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java new file mode 100644 index 0000000..5ccf471 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java @@ -0,0 +1,32 @@ +package com.arch.policy.aggregation; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public final class LocalSearchWaiters { + private final ConcurrentMap waiters = new ConcurrentHashMap(); + + public Waiter register(String searchKey) { + Waiter waiter = new Waiter(); + Waiter existing = waiters.putIfAbsent(searchKey, waiter); + if (existing != null) throw new IllegalStateException("duplicate local search: " + searchKey); + return waiter; + } + + public void remove(String searchKey, Waiter waiter) { waiters.remove(searchKey, waiter); } + + public void signal(String searchKey) { + Waiter waiter = waiters.get(searchKey); + if (waiter != null) waiter.signal(); + } + + public static final class Waiter { + private final CountDownLatch completed = new CountDownLatch(1); + public void await(long millis) throws InterruptedException { + completed.await(millis, TimeUnit.MILLISECONDS); + } + private void signal() { completed.countDown(); } + } +} diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SearchState.java b/architecture/src/main/java/com/arch/policy/aggregation/SearchState.java new file mode 100644 index 0000000..fa6ae2e --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/SearchState.java @@ -0,0 +1,3 @@ +package com.arch.policy.aggregation; + +public enum SearchState { WAITING, COMPLETED, TIMED_OUT } diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java b/architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java new file mode 100644 index 0000000..c0a0a33 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java @@ -0,0 +1,13 @@ +package com.arch.policy.aggregation; + +import java.util.List; +import java.util.Set; + +public interface SearchStateStore { + void initialize(String searchKey, Set supplierIds, long ttlSeconds); + SearchState getState(String searchKey); + List getResultPayloads(String searchKey); + void recordCallback(String searchKey, String supplierId, String payload, + boolean finished, long ttlSeconds); + void markTimedOut(String searchKey, long ttlSeconds); +} diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java b/architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java new file mode 100644 index 0000000..6113c36 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java @@ -0,0 +1,38 @@ +package com.arch.policy.aggregation; + +import com.arch.policy.api.SupplierCallbackRequest; +import com.fasterxml.jackson.databind.ObjectMapper; + +public final class SupplierCallbackService { + private final SearchStateStore stateStore; + private final ObjectMapper objectMapper; + private final long redisTtlSeconds; + + public SupplierCallbackService(SearchStateStore stateStore, ObjectMapper objectMapper, + long redisTtlSeconds) { + this.stateStore = stateStore; + this.objectMapper = objectMapper; + this.redisTtlSeconds = redisTtlSeconds; + } + + public void callback(SupplierCallbackRequest request) throws Exception { + validate(request); + String payload = request.getResults().isEmpty() ? "" : objectMapper.writeValueAsString(request); + stateStore.recordCallback(request.getSearchKey(), request.getSupplierId(), payload, + request.isSearchFinished(), redisTtlSeconds); + } + + public void markFinishedWithoutResult(String searchKey, String supplierId) throws Exception { + SupplierCallbackRequest request = new SupplierCallbackRequest(); + request.setSearchKey(searchKey); + request.setSupplierId(supplierId); + request.setSearchFinished(true); + callback(request); + } + + private static void validate(SupplierCallbackRequest request) { + if (request == null || request.getSearchKey() == null || request.getSupplierId() == null) { + throw new IllegalArgumentException("searchKey and supplierId are required"); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java b/architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java new file mode 100644 index 0000000..ab43db7 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java @@ -0,0 +1,9 @@ +package com.arch.policy.aggregation; + +import com.arch.policy.api.PolicySearchRequest; + +import java.util.Set; + +public interface SupplierTaskDispatcher { + void dispatch(String searchKey, Set supplierIds, PolicySearchRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/api/CallbackResponse.java b/architecture/src/main/java/com/arch/policy/api/CallbackResponse.java new file mode 100644 index 0000000..43b5166 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/CallbackResponse.java @@ -0,0 +1,10 @@ +package com.arch.policy.api; + +import java.io.Serializable; + +public final class CallbackResponse implements Serializable { + private static final long serialVersionUID = 1L; + private final boolean success; + public CallbackResponse(boolean success) { this.success = success; } + public boolean isSuccess() { return success; } +} diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java b/architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java new file mode 100644 index 0000000..fe9a308 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java @@ -0,0 +1,22 @@ +package com.arch.policy.api; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class PolicySearchRequest implements Serializable { + private static final long serialVersionUID = 1L; + private String criteria; + private List supplierIds = new ArrayList(); + private long totalTimeoutMillis = 3000; + + public String getCriteria() { return criteria; } + public void setCriteria(String criteria) { this.criteria = criteria; } + public List getSupplierIds() { return Collections.unmodifiableList(supplierIds); } + public void setSupplierIds(List supplierIds) { + this.supplierIds = supplierIds == null ? new ArrayList() : new ArrayList(supplierIds); + } + public long getTotalTimeoutMillis() { return totalTimeoutMillis; } + public void setTotalTimeoutMillis(long totalTimeoutMillis) { this.totalTimeoutMillis = totalTimeoutMillis; } +} diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java b/architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java new file mode 100644 index 0000000..f8b3ecd --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java @@ -0,0 +1,22 @@ +package com.arch.policy.api; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class PolicySearchResponse implements Serializable { + private static final long serialVersionUID = 1L; + private final String searchKey; + private final String state; + private final List results; + + public PolicySearchResponse(String searchKey, String state, List results) { + this.searchKey = searchKey; + this.state = state; + this.results = Collections.unmodifiableList(new ArrayList(results)); + } + public String getSearchKey() { return searchKey; } + public String getState() { return state; } + public List getResults() { return results; } +} diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java b/architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java new file mode 100644 index 0000000..5ab9e75 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java @@ -0,0 +1,7 @@ +package com.arch.policy.api; + +import java.util.concurrent.CompletableFuture; + +public interface PolicySearchRpcService { + CompletableFuture asyncSearch(PolicySearchRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java b/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java new file mode 100644 index 0000000..f965f51 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java @@ -0,0 +1,25 @@ +package com.arch.policy.api; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class SupplierCallbackRequest implements Serializable { + private static final long serialVersionUID = 1L; + private String searchKey; + private String supplierId; + private List results = new ArrayList(); + private boolean searchFinished; + + public String getSearchKey() { return searchKey; } + public void setSearchKey(String searchKey) { this.searchKey = searchKey; } + public String getSupplierId() { return supplierId; } + public void setSupplierId(String supplierId) { this.supplierId = supplierId; } + public List getResults() { return Collections.unmodifiableList(results); } + public void setResults(List results) { + this.results = results == null ? new ArrayList() : new ArrayList(results); + } + public boolean isSearchFinished() { return searchFinished; } + public void setSearchFinished(boolean searchFinished) { this.searchFinished = searchFinished; } +} diff --git a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java b/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java new file mode 100644 index 0000000..1cdff44 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java @@ -0,0 +1,5 @@ +package com.arch.policy.api; + +public interface SupplierCallbackRpcService { + CallbackResponse callback(SupplierCallbackRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java b/architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java new file mode 100644 index 0000000..d7d1d35 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java @@ -0,0 +1,16 @@ +package com.arch.policy.application; + +import com.arch.policy.ActiveSnapshotRegistry; +import com.arch.policy.PolicyChange; + +public final class PolicyIncrementalUpdater { + private final ActiveSnapshotRegistry registry; + + public PolicyIncrementalUpdater(ActiveSnapshotRegistry registry) { this.registry = registry; } + + public boolean apply(PolicyChange change) throws Exception { + try (ActiveSnapshotRegistry.SnapshotLease lease = registry.acquire()) { + return lease.snapshot().apply(change); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java b/architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java new file mode 100644 index 0000000..58bbf5f --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java @@ -0,0 +1,19 @@ +package com.arch.policy.application; + +import com.arch.policy.PolicySnapshotService; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; + +public final class PolicyStartupRunner implements ApplicationRunner { + private final PolicySnapshotService snapshotService; + private final String initialVersion; + + public PolicyStartupRunner(PolicySnapshotService snapshotService, String initialVersion) { + this.snapshotService = snapshotService; + this.initialVersion = initialVersion; + } + + @Override public void run(ApplicationArguments args) throws Exception { + snapshotService.initializeBlocking(initialVersion); + } +} diff --git a/architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java b/architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java new file mode 100644 index 0000000..03f770c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java @@ -0,0 +1,109 @@ +package com.arch.policy.config; + +import com.arch.policy.ActiveSnapshotRegistry; +import com.arch.policy.DefaultSnapshotValidator; +import com.arch.policy.FileSystemSnapshotDirectory; +import com.arch.policy.PolicySnapshotService; +import com.arch.policy.SnapshotBuilder; +import com.arch.policy.SnapshotPorts.FullPolicyLoader; +import com.arch.policy.SnapshotPorts.IncrementalReplayer; +import com.arch.policy.application.PolicyIncrementalUpdater; +import com.arch.policy.application.PolicyStartupRunner; +import com.arch.policy.aggregation.AsyncSearchCoordinator; +import com.arch.policy.aggregation.LocalSearchWaiters; +import com.arch.policy.aggregation.SearchStateStore; +import com.arch.policy.aggregation.SupplierCallbackService; +import com.arch.policy.aggregation.SupplierTaskDispatcher; +import com.arch.policy.demo.DemoSupplierTaskDispatcher; +import com.arch.policy.kafka.KafkaPolicyChangeListener; +import com.arch.policy.redis.RedisSearchStateStore; +import com.arch.policy.redis.SearchFinishedSubscriber; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.listener.ChannelTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.nio.file.Paths; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +@Configuration +public class PolicySearchConfiguration { + @Bean public ActiveSnapshotRegistry activeSnapshotRegistry() { return new ActiveSnapshotRegistry(); } + + @Bean public PolicyIncrementalUpdater policyIncrementalUpdater(ActiveSnapshotRegistry registry) { + return new PolicyIncrementalUpdater(registry); + } + + @Bean(destroyMethod = "shutdown") public Executor policySearchExecutor() { + return Executors.newFixedThreadPool(Math.max(2, Runtime.getRuntime().availableProcessors())); + } + + @Bean public ObjectMapper objectMapper() { return new ObjectMapper(); } + + @Bean public SearchStateStore searchStateStore(StringRedisTemplate redis) { + return new RedisSearchStateStore(redis); + } + + @Bean public LocalSearchWaiters localSearchWaiters() { return new LocalSearchWaiters(); } + + @Bean public SupplierCallbackService supplierCallbackService( + SearchStateStore store, ObjectMapper mapper, + @Value("${policy.search.redis-ttl-seconds:60}") long ttlSeconds) { + return new SupplierCallbackService(store, mapper, ttlSeconds); + } + + @Bean(destroyMethod = "shutdown") public ScheduledExecutorService demoSupplierExecutor() { + return Executors.newScheduledThreadPool(4); + } + + @Bean public SupplierTaskDispatcher supplierTaskDispatcher( + ScheduledExecutorService demoSupplierExecutor, SupplierCallbackService callbackService) { + return new DemoSupplierTaskDispatcher(demoSupplierExecutor, callbackService); + } + + @Bean public AsyncSearchCoordinator asyncSearchCoordinator( + SearchStateStore store, SupplierTaskDispatcher dispatcher, LocalSearchWaiters waiters, + ObjectMapper mapper, @Value("${policy.search.redis-ttl-seconds:60}") long ttlSeconds) { + return new AsyncSearchCoordinator(store, dispatcher, waiters, mapper, ttlSeconds); + } + + @Bean public RedisMessageListenerContainer searchFinishedListener( + RedisConnectionFactory connectionFactory, LocalSearchWaiters waiters) { + RedisMessageListenerContainer container = new RedisMessageListenerContainer(); + container.setConnectionFactory(connectionFactory); + container.addMessageListener(new SearchFinishedSubscriber(waiters), + new ChannelTopic(RedisSearchStateStore.FINISHED_CHANNEL)); + return container; + } + + @Bean public KafkaPolicyChangeListener kafkaPolicyChangeListener(ObjectMapper mapper, + PolicyIncrementalUpdater updater) { + return new KafkaPolicyChangeListener(mapper, updater); + } + + @Bean public SnapshotBuilder snapshotBuilder(FullPolicyLoader fullLoader, + IncrementalReplayer replayer, + @Value("${policy.snapshot.directory:./data/policy-snapshots}") String directory) { + return new SnapshotBuilder(fullLoader, replayer, new DefaultSnapshotValidator(), + new FileSystemSnapshotDirectory(Paths.get(directory))); + } + + @Bean(destroyMethod = "close") public PolicySnapshotService policySnapshotService( + SnapshotBuilder builder, ActiveSnapshotRegistry registry, + @Value("${policy.snapshot.retry-delay-ms:5000}") long retryDelayMillis) { + return new PolicySnapshotService(builder, registry, + Executors.newSingleThreadScheduledExecutor(), retryDelayMillis); + } + + @Bean public PolicyStartupRunner policyStartupRunner( + PolicySnapshotService service, + @Value("${policy.snapshot.initial-version:startup}") String initialVersion) { + return new PolicyStartupRunner(service, initialVersion); + } +} diff --git a/architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java b/architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java new file mode 100644 index 0000000..ecb3714 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java @@ -0,0 +1,47 @@ +package com.arch.policy.demo; + +import com.arch.policy.aggregation.SupplierCallbackService; +import com.arch.policy.aggregation.SupplierTaskDispatcher; +import com.arch.policy.api.PolicySearchRequest; +import com.arch.policy.api.SupplierCallbackRequest; + +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** Simulates downstream suppliers asynchronously calling this module back. */ +public final class DemoSupplierTaskDispatcher implements SupplierTaskDispatcher { + private final ScheduledExecutorService executor; + private final SupplierCallbackService callbackService; + + public DemoSupplierTaskDispatcher(ScheduledExecutorService executor, + SupplierCallbackService callbackService) { + this.executor = executor; + this.callbackService = callbackService; + } + + @Override public void dispatch(final String searchKey, Set supplierIds, + final PolicySearchRequest request) { + int delay = 20; + for (final String supplierId : supplierIds) { + executor.schedule(new Runnable() { + @Override public void run() { + try { + SupplierCallbackRequest callback = new SupplierCallbackRequest(); + callback.setSearchKey(searchKey); + callback.setSupplierId(supplierId); + callback.setResults(Collections.singletonList( + supplierId + " quote for " + request.getCriteria())); + callback.setSearchFinished(true); + callbackService.callback(callback); + } catch (Exception failure) { + try { callbackService.markFinishedWithoutResult(searchKey, supplierId); } + catch (Exception ignored) { /* broker retry/alert belongs in a real adapter */ } + } + } + }, delay, TimeUnit.MILLISECONDS); + delay += 20; + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java b/architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java new file mode 100644 index 0000000..595318c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java @@ -0,0 +1,45 @@ +package com.arch.policy.kafka; + +import com.arch.policy.MessagePosition; +import com.arch.policy.PolicyChange; +import com.arch.policy.PolicyRecord; +import com.arch.policy.application.PolicyIncrementalUpdater; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.kafka.annotation.KafkaListener; + +import java.nio.charset.StandardCharsets; +import java.util.HashSet; + +public final class KafkaPolicyChangeListener { + private final ObjectMapper objectMapper; + private final PolicyIncrementalUpdater updater; + + public KafkaPolicyChangeListener(ObjectMapper objectMapper, PolicyIncrementalUpdater updater) { + this.objectMapper = objectMapper; + this.updater = updater; + } + + @KafkaListener(topics = "${policy.kafka.topic:policy-change}", + groupId = "${policy.kafka.group-id:policy-search}") + public void onMessage(String json) throws Exception { + PolicyChangeMessage message = objectMapper.readValue(json, PolicyChangeMessage.class); + updater.apply(toChange(message)); + } + + private PolicyChange toChange(PolicyChangeMessage message) { + MessagePosition position = new MessagePosition(message.getPosition()); + if ("DELETE".equalsIgnoreCase(message.getOperation())) { + return PolicyChange.delete(message.getPolicyId(), position); + } + if (!"UPSERT".equalsIgnoreCase(message.getOperation())) { + throw new IllegalArgumentException("unsupported policy operation: " + message.getOperation()); + } + if (message.getDetail() == null || message.getIndexTerms() == null) { + throw new IllegalArgumentException("UPSERT requires detail and indexTerms"); + } + PolicyRecord policy = new PolicyRecord(message.getPolicyId(), + message.getDetail().getBytes(StandardCharsets.UTF_8), + new HashSet(message.getIndexTerms())); + return PolicyChange.upsert(policy, position); + } +} diff --git a/architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java b/architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java new file mode 100644 index 0000000..3f539ec --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java @@ -0,0 +1,23 @@ +package com.arch.policy.kafka; + +import java.util.ArrayList; +import java.util.List; + +public final class PolicyChangeMessage { + private String operation; + private int policyId; + private String detail; + private List indexTerms = new ArrayList(); + private long position; + + public String getOperation() { return operation; } + public void setOperation(String operation) { this.operation = operation; } + public int getPolicyId() { return policyId; } + public void setPolicyId(int policyId) { this.policyId = policyId; } + public String getDetail() { return detail; } + public void setDetail(String detail) { this.detail = detail; } + public List getIndexTerms() { return indexTerms; } + public void setIndexTerms(List indexTerms) { this.indexTerms = indexTerms; } + public long getPosition() { return position; } + public void setPosition(long position) { this.position = position; } +} diff --git a/architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java b/architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java new file mode 100644 index 0000000..49547f0 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java @@ -0,0 +1,84 @@ +package com.arch.policy.redis; + +import com.arch.policy.aggregation.SearchState; +import com.arch.policy.aggregation.SearchStateStore; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.DefaultRedisScript; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +public final class RedisSearchStateStore implements SearchStateStore { + public static final String FINISHED_CHANNEL = "search-finished"; + private static final String EMPTY_RESULT_MARKER = "__SEARCH_INITIALIZED__"; + private static final DefaultRedisScript INIT = script( + "redis.call('DEL', KEYS[1], KEYS[2], KEYS[3]); redis.call('RPUSH', KEYS[1], ARGV[2]); " + + "redis.call('EXPIRE', KEYS[1], ARGV[1]); " + + "for i=3,#ARGV do redis.call('SADD', KEYS[2], ARGV[i]); end; " + + "if redis.call('SCARD', KEYS[2]) == 0 then " + + " redis.call('SET', KEYS[3], 'COMPLETED', 'EX', ARGV[1]); return 1; " + + "end; redis.call('SET', KEYS[3], 'WAITING', 'EX', ARGV[1]); " + + "redis.call('EXPIRE', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[2], ARGV[1]); return 0;"); + private static final DefaultRedisScript CALLBACK = script( + "if redis.call('GET', KEYS[3]) ~= 'WAITING' then return 0; end; " + + "if redis.call('SISMEMBER', KEYS[2], ARGV[2]) == 0 then return 0; end; " + + "if ARGV[1] ~= '' then redis.call('RPUSH', KEYS[1], ARGV[1]); end; " + + "local removed=0; if ARGV[3] == 'true' then removed=redis.call('SREM', KEYS[2], ARGV[2]); end; " + + "redis.call('EXPIRE', KEYS[1], ARGV[5]); redis.call('EXPIRE', KEYS[2], ARGV[5]); " + + "redis.call('EXPIRE', KEYS[3], ARGV[5]); " + + "if removed > 0 and redis.call('SCARD', KEYS[2]) == 0 then " + + " redis.call('SET', KEYS[3], 'COMPLETED', 'EX', ARGV[5]); " + + " redis.call('PUBLISH', ARGV[6], ARGV[4]); return 2; end; return 1;"); + private static final DefaultRedisScript TIMEOUT = script( + "if redis.call('GET', KEYS[3]) == 'WAITING' then " + + " redis.call('SET', KEYS[3], 'TIMED_OUT', 'EX', ARGV[1]); " + + " redis.call('EXPIRE', KEYS[1], ARGV[1]); redis.call('EXPIRE', KEYS[2], ARGV[1]); return 1; end; return 0;"); + + private final StringRedisTemplate redis; + + public RedisSearchStateStore(StringRedisTemplate redis) { this.redis = redis; } + + @Override public void initialize(String searchKey, Set supplierIds, long ttlSeconds) { + List args = new ArrayList(); + args.add(Long.toString(ttlSeconds)); + args.add(EMPTY_RESULT_MARKER); + args.addAll(supplierIds); + redis.execute(INIT, keys(searchKey), args.toArray()); + } + + @Override public SearchState getState(String searchKey) { + String state = redis.opsForValue().get(stateKey(searchKey)); + return state == null ? SearchState.TIMED_OUT : SearchState.valueOf(state); + } + + @Override public List getResultPayloads(String searchKey) { + List values = redis.opsForList().range(resultKey(searchKey), 0, -1); + if (values == null) return Collections.emptyList(); + List results = new ArrayList(values); + results.remove(EMPTY_RESULT_MARKER); + return results; + } + + @Override public void recordCallback(String searchKey, String supplierId, String payload, + boolean finished, long ttlSeconds) { + redis.execute(CALLBACK, keys(searchKey), payload, supplierId, Boolean.toString(finished), + searchKey, Long.toString(ttlSeconds), FINISHED_CHANNEL); + } + + @Override public void markTimedOut(String searchKey, long ttlSeconds) { + redis.execute(TIMEOUT, keys(searchKey), Long.toString(ttlSeconds)); + } + + private static List keys(String searchKey) { + return Arrays.asList(resultKey(searchKey), pendingKey(searchKey), stateKey(searchKey)); + } + private static String resultKey(String key) { return "search:" + key + ":results"; } + private static String pendingKey(String key) { return "search:" + key + ":pending"; } + private static String stateKey(String key) { return "search:" + key + ":state"; } + private static DefaultRedisScript script(String source) { + return new DefaultRedisScript(source, Long.class); + } +} diff --git a/architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java b/architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java new file mode 100644 index 0000000..f8ea514 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java @@ -0,0 +1,15 @@ +package com.arch.policy.redis; + +import com.arch.policy.aggregation.LocalSearchWaiters; +import org.springframework.data.redis.connection.Message; +import org.springframework.data.redis.connection.MessageListener; + +import java.nio.charset.StandardCharsets; + +public final class SearchFinishedSubscriber implements MessageListener { + private final LocalSearchWaiters waiters; + public SearchFinishedSubscriber(LocalSearchWaiters waiters) { this.waiters = waiters; } + @Override public void onMessage(Message message, byte[] pattern) { + waiters.signal(new String(message.getBody(), StandardCharsets.UTF_8)); + } +} diff --git a/architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java b/architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java new file mode 100644 index 0000000..ef51aa6 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java @@ -0,0 +1,34 @@ +package com.arch.policy.rpc; + +import com.arch.policy.api.PolicySearchRequest; +import com.arch.policy.api.PolicySearchResponse; +import com.arch.policy.api.PolicySearchRpcService; +import com.arch.policy.aggregation.AsyncSearchCoordinator; +import org.apache.dubbo.config.annotation.DubboService; +import org.springframework.beans.factory.annotation.Qualifier; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; + +@DubboService(version = "1.0.0", timeout = 3000) +public final class DubboPolicySearchService implements PolicySearchRpcService { + private final AsyncSearchCoordinator searchService; + private final Executor searchExecutor; + + public DubboPolicySearchService(AsyncSearchCoordinator searchService, + @Qualifier("policySearchExecutor") Executor searchExecutor) { + this.searchService = searchService; + this.searchExecutor = searchExecutor; + } + + @Override public CompletableFuture asyncSearch(final PolicySearchRequest request) { + final CompletableFuture future = new CompletableFuture(); + searchExecutor.execute(new Runnable() { + @Override public void run() { + try { future.complete(searchService.search(request)); } + catch (Exception failure) { future.completeExceptionally(failure); } + } + }); + return future; + } +} diff --git a/architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java b/architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java new file mode 100644 index 0000000..f685dc1 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java @@ -0,0 +1,23 @@ +package com.arch.policy.rpc; + +import com.arch.policy.aggregation.SupplierCallbackService; +import com.arch.policy.api.CallbackResponse; +import com.arch.policy.api.SupplierCallbackRequest; +import com.arch.policy.api.SupplierCallbackRpcService; +import org.apache.dubbo.config.annotation.DubboService; + +@DubboService(version = "1.0.0", timeout = 3000) +public final class DubboSupplierCallbackService implements SupplierCallbackRpcService { + private final SupplierCallbackService callbackService; + public DubboSupplierCallbackService(SupplierCallbackService callbackService) { + this.callbackService = callbackService; + } + @Override public CallbackResponse callback(SupplierCallbackRequest request) { + try { + callbackService.callback(request); + return new CallbackResponse(true); + } catch (Exception failure) { + throw new IllegalStateException("supplier callback failed", failure); + } + } +} diff --git a/architecture/src/main/resources/application.yml b/architecture/src/main/resources/application.yml new file mode 100644 index 0000000..c30aadb --- /dev/null +++ b/architecture/src/main/resources/application.yml @@ -0,0 +1,32 @@ +spring: + application: + name: policy-search + kafka: + consumer: + enable-auto-commit: false + auto-offset-reset: earliest + key-deserializer: org.apache.kafka.common.serialization.StringDeserializer + value-deserializer: org.apache.kafka.common.serialization.StringDeserializer + listener: + ack-mode: record + redis: + host: localhost + port: 6379 + +dubbo: + application: + name: policy-search + protocol: + name: dubbo + port: 20880 + +policy: + search: + redis-ttl-seconds: 60 + snapshot: + directory: ./data/policy-snapshots + initial-version: startup + retry-delay-ms: 5000 + kafka: + topic: policy-change + group-id: policy-search diff --git a/architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java b/architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java new file mode 100644 index 0000000..fe28969 --- /dev/null +++ b/architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java @@ -0,0 +1,99 @@ +package com.arch.policy; + +import com.arch.policy.aggregation.*; +import com.arch.policy.api.PolicySearchRequest; +import com.arch.policy.api.PolicySearchResponse; +import com.arch.policy.api.SupplierCallbackRequest; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.*; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class AsyncSearchCoordinatorTest { + @Test + void waitsForAllSuppliersAndAggregatesAsyncCallbacks() throws Exception { + final MemoryStore store = new MemoryStore(); + final LocalSearchWaiters waiters = new LocalSearchWaiters(); + final ObjectMapper mapper = new ObjectMapper(); + SupplierTaskDispatcher dispatcher = (searchKey, suppliers, request) -> { + new Thread(() -> { + try { + for (String supplier : suppliers) { + SupplierCallbackRequest callback = callback(searchKey, supplier, supplier + "-quote", true); + store.recordCallback(searchKey, supplier, mapper.writeValueAsString(callback), true, 60); + } + waiters.signal(searchKey); + } catch (Exception failure) { throw new RuntimeException(failure); } + }).start(); + }; + AsyncSearchCoordinator coordinator = new AsyncSearchCoordinator(store, dispatcher, waiters, mapper, 60); + PolicySearchRequest request = request(1000, "A", "B"); + + PolicySearchResponse response = coordinator.search(request); + + assertEquals(SearchState.COMPLETED.name(), response.getState()); + assertEquals(2, response.getResults().size()); + } + + @Test + void timeoutReturnsPartialResultsAndRejectsLateCallbacks() throws Exception { + final MemoryStore store = new MemoryStore(); + final ObjectMapper mapper = new ObjectMapper(); + SupplierTaskDispatcher dispatcher = (searchKey, suppliers, request) -> { + try { + SupplierCallbackRequest partial = callback(searchKey, "A", "A-quote", true); + store.recordCallback(searchKey, "A", mapper.writeValueAsString(partial), true, 60); + } catch (Exception failure) { throw new RuntimeException(failure); } + }; + AsyncSearchCoordinator coordinator = new AsyncSearchCoordinator( + store, dispatcher, new LocalSearchWaiters(), mapper, 60); + + PolicySearchResponse response = coordinator.search(request(30, "A", "B")); + + assertEquals(SearchState.TIMED_OUT.name(), response.getState()); + assertEquals(1, response.getResults().size()); + } + + private static PolicySearchRequest request(long timeout, String... suppliers) { + PolicySearchRequest request = new PolicySearchRequest(); + request.setCriteria("demo"); + request.setSupplierIds(Arrays.asList(suppliers)); + request.setTotalTimeoutMillis(timeout); + return request; + } + + private static SupplierCallbackRequest callback(String key, String supplier, String result, boolean done) { + SupplierCallbackRequest request = new SupplierCallbackRequest(); + request.setSearchKey(key); + request.setSupplierId(supplier); + request.setResults(Collections.singletonList(result)); + request.setSearchFinished(done); + return request; + } + + private static final class MemoryStore implements SearchStateStore { + private SearchState state; + private final Set pending = new HashSet(); + private final List results = new ArrayList(); + @Override public synchronized void initialize(String key, Set suppliers, long ttl) { + pending.addAll(suppliers); + state = suppliers.isEmpty() ? SearchState.COMPLETED : SearchState.WAITING; + } + @Override public synchronized SearchState getState(String key) { return state; } + @Override public synchronized List getResultPayloads(String key) { + return new ArrayList(results); + } + @Override public synchronized void recordCallback(String key, String supplier, String payload, + boolean finished, long ttl) { + if (state != SearchState.WAITING) return; + if (!payload.isEmpty()) results.add(payload); + if (finished) pending.remove(supplier); + if (pending.isEmpty()) state = SearchState.COMPLETED; + } + @Override public synchronized void markTimedOut(String key, long ttl) { + if (state == SearchState.WAITING) state = SearchState.TIMED_OUT; + } + } +} diff --git a/architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java b/architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java new file mode 100644 index 0000000..297a7c0 --- /dev/null +++ b/architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java @@ -0,0 +1,119 @@ +package com.arch.policy; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import static com.arch.policy.SnapshotPorts.*; +import static org.junit.jupiter.api.Assertions.*; + +class SnapshotLifecycleTest { + @TempDir Path root; + + @Test + void buildsFullDataReplaysChangesAndValidatesPosition() throws Exception { + SnapshotBuilder builder = builder(false); + + PolicySnapshot snapshot = builder.build("V101"); + + assertEquals(new MessagePosition(12), snapshot.getPosition()); + assertArrayEquals(bytes("new-detail"), snapshot.findDetail(1)); + assertEquals(1, snapshot.findPolicyIds("active").getCardinality()); + assertEquals(0, snapshot.findPolicyIds("draft").getCardinality()); + snapshot.closeAndDelete(); + } + + @Test + void switchWaitsForOldQueriesBeforeClosingAndDeletingOldVersion() throws Exception { + ActiveSnapshotRegistry registry = new ActiveSnapshotRegistry(); + PolicySnapshot old = builder(false).build("V100"); + PolicySnapshot replacement = builder(false).build("V101"); + registry.activate(old); + ActiveSnapshotRegistry.SnapshotLease oldQuery = registry.acquire(); + + registry.activate(replacement); + + assertTrue(Files.exists(old.getDirectory())); + assertArrayEquals(bytes("new-detail"), oldQuery.snapshot().findDetail(1)); + try (ActiveSnapshotRegistry.SnapshotLease newQuery = registry.acquire()) { + assertEquals("V101", newQuery.snapshot().getVersion()); + } + oldQuery.close(); + assertFalse(Files.exists(old.getDirectory())); + assertThrows(IllegalStateException.class, () -> old.findPolicyIds("active")); + registry.close(); + } + + @Test + void failedRuntimeCandidateDoesNotReplaceServingSnapshot() throws Exception { + ActiveSnapshotRegistry registry = new ActiveSnapshotRegistry(); + registry.activate(builder(false).build("V100")); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + PolicySnapshotService service = new PolicySnapshotService(builder(true), registry, executor, 1); + + assertTrue(service.refresh("V101")); + waitUntilBuildFinishes(service); + + assertTrue(service.isReady()); + try (ActiveSnapshotRegistry.SnapshotLease query = registry.acquire()) { + assertEquals("V100", query.snapshot().getVersion()); + } + assertFalse(Files.exists(root.resolve("V101"))); + service.close(); + } + + private SnapshotBuilder builder(final boolean failValidation) { + FullPolicyLoader full = target -> { + target.upsert(new PolicyRecord(1, bytes("old-detail"), Collections.singleton("draft"))); + return new MessagePosition(10); + }; + IncrementalReplayer replay = new IncrementalReplayer() { + @Override public MessagePosition latestPosition() { return new MessagePosition(12); } + @Override public MessagePosition replayInto(PolicySnapshot target, MessagePosition from, + MessagePosition latest) throws Exception { + target.upsert(new PolicyRecord(1, bytes("new-detail"), + Collections.singleton("active"))); + return latest; + } + }; + SnapshotValidator validator = (candidate, position) -> { + new DefaultSnapshotValidator().validate(candidate, position); + if (failValidation) throw new IllegalStateException("invalid candidate"); + }; + SnapshotDirectory directories = new SnapshotDirectory() { + @Override public Path create(String version) throws Exception { + Path path = root.resolve(version); + Files.createDirectories(path); + return path; + } + @Override public void delete(Path directory) throws Exception { + if (!Files.exists(directory)) return; + Files.walk(directory).sorted(Comparator.reverseOrder()).forEach(path -> { + try { Files.deleteIfExists(path); } + catch (Exception failure) { throw new RuntimeException(failure); } + }); + } + }; + return new SnapshotBuilder(full, replay, validator, directories); + } + + private static byte[] bytes(String value) { + return value.getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + private static void waitUntilBuildFinishes(PolicySnapshotService service) throws Exception { + long deadline = System.currentTimeMillis() + 3000; + while (System.currentTimeMillis() < deadline) { + if (!service.isBuilding()) return; + Thread.sleep(10); + } + fail("candidate build did not finish"); + } +} From 5deeb99325653b946b380e7d854a735b1a809965 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Sun, 19 Jul 2026 21:12:39 +0800 Subject: [PATCH 33/37] =?UTF-8?q?=E5=AE=8C=E5=96=84=E4=BC=AA=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=9E=B6=E6=9E=84=E5=BA=93=E5=88=86=E5=B1=82=E7=BB=93?= =?UTF-8?q?=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture/README.md | 214 ++++++++++++++---- .../api/SupplierCallbackRpcService.java | 5 - .../{ => search}/PolicySearchRpcService.java | 5 +- .../search/SupplierCallbackRpcService.java | 8 + .../config/PolicySearchConfiguration.java | 40 ++-- .../{ => common/model}/MessagePosition.java | 2 +- .../{ => common/model}/PolicyChange.java | 2 +- .../{ => common/model}/PolicyRecord.java | 2 +- .../search}/CallbackResponse.java | 2 +- .../search}/PolicySearchRequest.java | 2 +- .../search}/PolicySearchResponse.java | 2 +- .../search}/SupplierCallbackRequest.java | 2 +- .../application}/AsyncSearchCoordinator.java | 8 +- .../application}/LocalSearchWaiters.java | 2 +- .../application/PolicyIncrementalUpdater.java | 6 +- .../application/PolicyStartupRunner.java | 4 +- .../application}/SearchState.java | 2 +- .../application}/SearchStateStore.java | 2 +- .../application}/SupplierCallbackService.java | 4 +- .../application}/SupplierTaskDispatcher.java | 4 +- .../snapshot}/ActiveSnapshotRegistry.java | 2 +- .../snapshot}/DefaultSnapshotValidator.java | 6 +- .../FileSystemSnapshotDirectory.java | 4 +- .../domain/snapshot}/PolicySnapshot.java | 6 +- .../snapshot}/PolicySnapshotService.java | 2 +- .../domain/snapshot}/SnapshotBuilder.java | 6 +- .../domain/snapshot}/SnapshotPorts.java | 4 +- .../demo/DemoSupplierTaskDispatcher.java | 10 +- .../kafka/KafkaPolicyChangeListener.java | 10 +- .../kafka/PolicyChangeMessage.java | 2 +- .../redis/RedisSearchStateStore.java | 6 +- .../redis/SearchFinishedSubscriber.java | 4 +- .../rpc/DubboPolicySearchService.java | 10 +- .../rpc/DubboSupplierCallbackService.java | 10 +- .../AsyncSearchCoordinatorTest.java | 9 +- .../snapshot}/SnapshotLifecycleTest.java | 6 +- 36 files changed, 277 insertions(+), 138 deletions(-) delete mode 100644 architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java rename architecture/src/main/java/com/arch/policy/api/{ => search}/PolicySearchRpcService.java (53%) create mode 100644 architecture/src/main/java/com/arch/policy/api/search/SupplierCallbackRpcService.java rename architecture/src/main/java/com/arch/policy/{ => common}/config/PolicySearchConfiguration.java (77%) rename architecture/src/main/java/com/arch/policy/{ => common/model}/MessagePosition.java (95%) rename architecture/src/main/java/com/arch/policy/{ => common/model}/PolicyChange.java (96%) rename architecture/src/main/java/com/arch/policy/{ => common/model}/PolicyRecord.java (95%) rename architecture/src/main/java/com/arch/policy/{api => common/search}/CallbackResponse.java (88%) rename architecture/src/main/java/com/arch/policy/{api => common/search}/PolicySearchRequest.java (96%) rename architecture/src/main/java/com/arch/policy/{api => common/search}/PolicySearchResponse.java (95%) rename architecture/src/main/java/com/arch/policy/{api => common/search}/SupplierCallbackRequest.java (96%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/AsyncSearchCoordinator.java (93%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/LocalSearchWaiters.java (96%) rename architecture/src/main/java/com/arch/policy/{ => search}/application/PolicyIncrementalUpdater.java (71%) rename architecture/src/main/java/com/arch/policy/{ => search}/application/PolicyStartupRunner.java (84%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/SearchState.java (57%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/SearchStateStore.java (91%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/SupplierCallbackService.java (93%) rename architecture/src/main/java/com/arch/policy/{aggregation => search/application}/SupplierTaskDispatcher.java (61%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/ActiveSnapshotRegistry.java (97%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/DefaultSnapshotValidator.java (76%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/FileSystemSnapshotDirectory.java (91%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/PolicySnapshot.java (96%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/PolicySnapshotService.java (98%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/SnapshotBuilder.java (92%) rename architecture/src/main/java/com/arch/policy/{ => search/domain/snapshot}/SnapshotPorts.java (90%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/demo/DemoSupplierTaskDispatcher.java (86%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/kafka/KafkaPolicyChangeListener.java (86%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/kafka/PolicyChangeMessage.java (94%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/redis/RedisSearchStateStore.java (96%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/redis/SearchFinishedSubscriber.java (82%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/rpc/DubboPolicySearchService.java (80%) rename architecture/src/main/java/com/arch/policy/{ => search/infrastructure}/rpc/DubboSupplierCallbackService.java (70%) rename architecture/src/test/java/com/arch/policy/{ => search/application}/AsyncSearchCoordinatorTest.java (95%) rename architecture/src/test/java/com/arch/policy/{ => search/domain/snapshot}/SnapshotLifecycleTest.java (95%) diff --git a/architecture/README.md b/architecture/README.md index ed7e14c..58cde70 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -1,46 +1,168 @@ -# Policy snapshot lifecycle - -This module implements version-isolated policy snapshots. Each snapshot owns a RocksDB detail -store, a bitmap index, and the incremental-message position at which both were built. - -## Integration flow - -1. Implement `FullPolicyLoader` to stream the full-data cut into `PolicySnapshot.upsert` and - return the cut's message position. -2. Implement `IncrementalReplayer` to apply upserts/deletes after that position. The builder - repeats replay until it reaches a stable latest position. -3. Supply `DefaultSnapshotValidator` (or a stricter domain validator), a - `FileSystemSnapshotDirectory`, and construct `SnapshotBuilder`. -4. Call `PolicySnapshotService.start(version)` during startup. Keep the application's readiness - probe bound to `service.isReady()`; failures remain unready and retry. -5. Call `refresh(newVersion)` at runtime. A failed candidate is discarded. A valid candidate is - atomically activated while the old snapshot continues serving existing leases. -6. Every search must use `try (SnapshotLease lease = registry.acquire())`. Releasing the last old - lease closes RocksDB and deletes that retired version's directory. - -The module intentionally leaves message-broker and full-data-source clients behind interfaces so -the snapshot consistency rules are independent of Kafka, HTTP, database, or framework choices. - -## Package layout - -- `api`: serializable Dubbo contract and request/response DTOs. -- `rpc`: Dubbo search provider and supplier callback provider. -- `aggregation`: Redis-backed fan-out/fan-in search coordination and local waiters. -- `redis`: atomic Lua state transitions and Pub/Sub early wake-up. -- `demo`: asynchronous downstream supplier simulation. -- `kafka`: JSON policy-change consumer and message DTO. -- `application`: startup and incremental-update use cases. -- root `policy` package: versioned RocksDB/Bitmap snapshot domain and lifecycle. - -Kafka messages use a globally monotonic `position` so duplicate/out-of-order delivery is ignored. -If the topic has multiple partitions, the producer must supply this global sequence; otherwise the -position model should be replaced with a per-partition offset map. - -## Async supplier search - -`asyncSearch` initializes pending suppliers, state and result TTL atomically in Redis, registers a -local waiter, double-checks Redis, then dispatches all supplier tasks. Supplier callbacks append -result chunks and remove a supplier from the pending set only on its final callback. The Lua script -sets `COMPLETED` and publishes `search-finished` when the last supplier finishes. Pub/Sub only wakes -the local waiter early; Redis remains the source of truth and is checked every 200 ms. Timeout is -also a Lua state transition and returns all partial results already recorded. +# 架构方案伪代码库 + +`architecture` module 用于沉淀不同业务场景下的架构思路、设计方案和关键伪代码。它关注的是 +复杂问题如何拆分、核心流程如何协作、数据一致性如何保证,以及不同技术组件之间的边界,而不是提供 +可以直接上线的完整业务系统。 + +当前实现以机票业务为背景,模拟以下两个相互关联的架构场景: + +1. **多供应商并发搜索**:一次机票查询并发分发给多个供应商,通过异步回调聚合结果,并支持超时返回部分结果。 +2. **海量政策高性能匹配**:使用 RocksDB 保存政策明细、RoaringBitmap 构建匹配索引,并通过版本化快照实现运行时无损切换。 + +后续会继续增加交易、下单等其他场景的架构伪代码,使该 module 逐步形成可复用的架构与设计方案集合。 + +## 模块目标 + +- 用最少的代码表达架构中的核心职责、协作关系和一致性约束。 +- 为类似业务问题提供可讨论、可验证、可演进的设计参考。 +- 通过接口隔离外部系统,使方案不被 Dubbo、Kafka、Redis 等具体技术绑定。 +- 通过测试验证关键架构行为,而不是覆盖完整的生产业务流程。 +- 持续积累搜索、政策匹配、交易、下单等不同场景的方案。 + +本模块属于架构伪代码,生产落地时仍需根据实际情况补充鉴权、限流、熔断、监控、链路追踪、异常分级、 +数据迁移、容量评估、容灾和部署方案。 + +## 场景目录 + +| 场景 | 核心问题 | 当前状态 | +| --- | --- | --- | +| 多供应商并发搜索 | 任务分发、异步回调、结果聚合、超时与迟到回调 | 已实现伪代码 | +| 海量政策匹配 | 全量加载、增量追平、位图索引、快照校验与热切换 | 已实现伪代码 | +| 交易/下单 | 幂等、状态流转、库存确认、分布式一致性与补偿 | 待补充 | + +## 分层结构 + +代码先区分项目级公开契约、公共能力和具体业务场景,再在场景内部按照应用、领域和基础设施分层。 + +```text +com.arch.policy +├── api +│ └── search # 项目级搜索契约、请求和响应 DTO +├── common +│ ├── config # Spring Bean 装配 +│ └── model # 可跨场景复用的模型 +├── search +│ ├── application # 查询编排、回调处理、启动和增量更新用例 +│ ├── domain +│ │ └── snapshot # RocksDB/位图快照领域逻辑 +│ └── infrastructure +│ ├── demo # 模拟供应商 +│ ├── kafka # 政策变更消息适配器 +│ ├── redis # 聚合状态和完成通知适配器 +│ └── rpc # Dubbo 接口实现 +└── PolicySearchApplication # 当前搜索场景的启动入口 +``` + +依赖方向如下: + +```mermaid +flowchart LR + Caller["外部调用方"] --> API["api.search
项目级公开契约"] + Infrastructure["search.infrastructure
RPC / Redis / Kafka"] --> API + Infrastructure --> Application["search.application
流程编排"] + Application --> Domain["search.domain
领域规则"] + Application --> Common["common
公共配置与模型"] + Domain --> Common +``` + +公开 API 不依赖场景内部实现,领域层不依赖 Dubbo、Redis、Kafka 等外部技术。后续增加交易场景时, +可以平行新增 `api.order` 和 `order.application/domain/infrastructure`,避免交易代码与搜索代码混杂。 +只有真正跨场景稳定复用的模型或能力才应放入 `common`。 + +## 场景一:多供应商并发搜索 + +查询服务将一次请求分发给多个供应商。供应商可以分批回调结果,最后一次回调负责声明该供应商完成。 +Redis 保存聚合结果、待完成供应商集合和查询状态,是整个流程的唯一事实来源。 + +```mermaid +sequenceDiagram + autonumber + actor Caller as 调用方 + participant SearchAPI as PolicySearchRpcService + participant Coordinator as AsyncSearchCoordinator + participant Redis as RedisSearchStateStore + participant Dispatcher as SupplierTaskDispatcher + participant Supplier as 多个供应商 + participant CallbackAPI as SupplierCallbackRpcService + participant Callback as SupplierCallbackService + participant Waiter as LocalSearchWaiters + + Caller->>SearchAPI: asyncSearch(request) + SearchAPI->>Coordinator: 执行查询 + Coordinator->>Redis: 初始化状态、待完成集合和 TTL + Coordinator->>Waiter: 注册本地等待器 + Coordinator->>Dispatcher: 并发分发供应商任务 + Dispatcher-->>Supplier: 异步查询 + + loop 供应商分批返回结果 + Supplier->>CallbackAPI: callback(result, searchFinished) + CallbackAPI->>Callback: 处理回调 + Callback->>Redis: 追加结果并更新待完成集合 + end + + Redis-->>Waiter: 全部完成后发布通知 + Waiter-->>Coordinator: 提前唤醒 + Coordinator->>Redis: 读取最终状态和结果 + Coordinator-->>SearchAPI: PolicySearchResponse + SearchAPI-->>Caller: 完成异步调用 +``` + +本地等待器和 Redis Pub/Sub 只用于提前唤醒。即使通知丢失,查询线程仍会定期检查 Redis。超过总超时时间后, +Lua 脚本会原子地将查询状态更新为 `TIMED_OUT`,返回已经收到的部分结果,并拒绝迟到回调继续修改结果。 + +## 场景二:海量政策匹配与快照切换 + +每个政策快照使用独立目录保存 RocksDB 明细库,并在内存中维护 RoaringBitmap 匹配索引。新版本先在旁路 +完成全量加载、增量追平和一致性校验,只有通过校验后才会替换当前服务版本。 + +```mermaid +flowchart TD + Start["应用启动或运行时刷新"] --> Create["创建版本隔离目录"] + Create --> Full["加载全量政策"] + Full --> Position["记录全量数据切点"] + Position --> Replay["回放切点后的增量消息"] + Replay --> CaughtUp{"已追平最新位置?"} + CaughtUp -- 否 --> Replay + CaughtUp -- 是 --> Validate["校验明细、索引和消息位置"] + Validate --> Valid{"校验成功?"} + Valid -- 否 --> Discard["关闭并删除候选快照"] + Valid -- 是 --> Activate["原子激活新快照"] + Activate --> NewQuery["新查询使用新版本"] + Activate --> Drain["旧版本等待已有查询释放租约"] + Drain --> Delete["关闭并删除旧快照"] +``` + +查询通过 `ActiveSnapshotRegistry.acquire()` 获取快照租约,并使用 `try-with-resources` 释放。切换完成后, +新查询立即使用新快照,旧快照则保留到最后一个旧查询结束,从而避免正在执行的查询访问已关闭的 RocksDB。 + +快照方案的主要扩展点: + +1. `FullPolicyLoader`:加载全量政策,并返回全量数据对应的消息位置。 +2. `IncrementalReplayer`:回放全量切点之后的新增、修改和删除事件。 +3. `SnapshotValidator`:校验政策明细、位图索引和消息位置的一致性。 +4. `SnapshotDirectory`:隔离不同快照版本的存储目录。 +5. `PolicySnapshotService`:负责首次初始化、失败重试和运行时刷新。 + +Kafka 政策变更消息使用全局单调递增的 `position`,重复或乱序消息会被忽略。如果 Topic 使用多个分区, +生产端必须提供全局序列;否则应将当前位置模型调整为按分区保存 offset。 + +## 新增架构场景的约定 + +新增交易、下单或其他架构伪代码时,应遵循以下约定: + +1. 在 README 的场景目录中说明要解决的问题、关键约束和方案状态。 +2. 对外契约放在 `api.`,内部实现放在对应的 `` 业务包。 +3. 使用 `application/domain/infrastructure` 表达职责边界,不让领域规则依赖具体中间件。 +4. 优先提供展示关键协作关系的最小实现,避免把伪代码扩展成不完整的生产框架。 +5. 为幂等、一致性、并发、超时、切换和失败隔离等关键架构行为编写测试。 +6. 在场景文档中记录设计取舍、适用边界,以及生产落地仍需补充的能力。 + +## 运行验证 + +在仓库根目录执行: + +```bash +mvn -f architecture/pom.xml clean test +``` + +当前测试覆盖异步供应商聚合、超时返回部分结果、全量加增量快照构建、运行时快照切换以及失败候选版本隔离。 diff --git a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java b/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java deleted file mode 100644 index 1cdff44..0000000 --- a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRpcService.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.api; - -public interface SupplierCallbackRpcService { - CallbackResponse callback(SupplierCallbackRequest request); -} diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java b/architecture/src/main/java/com/arch/policy/api/search/PolicySearchRpcService.java similarity index 53% rename from architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java rename to architecture/src/main/java/com/arch/policy/api/search/PolicySearchRpcService.java index 5ab9e75..98533cf 100644 --- a/architecture/src/main/java/com/arch/policy/api/PolicySearchRpcService.java +++ b/architecture/src/main/java/com/arch/policy/api/search/PolicySearchRpcService.java @@ -1,4 +1,7 @@ -package com.arch.policy.api; +package com.arch.policy.api.search; + +import com.arch.policy.common.search.PolicySearchRequest; +import com.arch.policy.common.search.PolicySearchResponse; import java.util.concurrent.CompletableFuture; diff --git a/architecture/src/main/java/com/arch/policy/api/search/SupplierCallbackRpcService.java b/architecture/src/main/java/com/arch/policy/api/search/SupplierCallbackRpcService.java new file mode 100644 index 0000000..8995d25 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/search/SupplierCallbackRpcService.java @@ -0,0 +1,8 @@ +package com.arch.policy.api.search; + +import com.arch.policy.common.search.CallbackResponse; +import com.arch.policy.common.search.SupplierCallbackRequest; + +public interface SupplierCallbackRpcService { + CallbackResponse callback(SupplierCallbackRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/PolicySearchConfiguration.java similarity index 77% rename from architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java rename to architecture/src/main/java/com/arch/policy/common/config/PolicySearchConfiguration.java index 03f770c..21191fb 100644 --- a/architecture/src/main/java/com/arch/policy/config/PolicySearchConfiguration.java +++ b/architecture/src/main/java/com/arch/policy/common/config/PolicySearchConfiguration.java @@ -1,23 +1,23 @@ -package com.arch.policy.config; - -import com.arch.policy.ActiveSnapshotRegistry; -import com.arch.policy.DefaultSnapshotValidator; -import com.arch.policy.FileSystemSnapshotDirectory; -import com.arch.policy.PolicySnapshotService; -import com.arch.policy.SnapshotBuilder; -import com.arch.policy.SnapshotPorts.FullPolicyLoader; -import com.arch.policy.SnapshotPorts.IncrementalReplayer; -import com.arch.policy.application.PolicyIncrementalUpdater; -import com.arch.policy.application.PolicyStartupRunner; -import com.arch.policy.aggregation.AsyncSearchCoordinator; -import com.arch.policy.aggregation.LocalSearchWaiters; -import com.arch.policy.aggregation.SearchStateStore; -import com.arch.policy.aggregation.SupplierCallbackService; -import com.arch.policy.aggregation.SupplierTaskDispatcher; -import com.arch.policy.demo.DemoSupplierTaskDispatcher; -import com.arch.policy.kafka.KafkaPolicyChangeListener; -import com.arch.policy.redis.RedisSearchStateStore; -import com.arch.policy.redis.SearchFinishedSubscriber; +package com.arch.policy.common.config; + +import com.arch.policy.search.application.AsyncSearchCoordinator; +import com.arch.policy.search.application.LocalSearchWaiters; +import com.arch.policy.search.application.PolicyIncrementalUpdater; +import com.arch.policy.search.application.PolicyStartupRunner; +import com.arch.policy.search.application.SearchStateStore; +import com.arch.policy.search.application.SupplierCallbackService; +import com.arch.policy.search.application.SupplierTaskDispatcher; +import com.arch.policy.search.domain.snapshot.ActiveSnapshotRegistry; +import com.arch.policy.search.domain.snapshot.DefaultSnapshotValidator; +import com.arch.policy.search.domain.snapshot.FileSystemSnapshotDirectory; +import com.arch.policy.search.domain.snapshot.PolicySnapshotService; +import com.arch.policy.search.domain.snapshot.SnapshotBuilder; +import com.arch.policy.search.domain.snapshot.SnapshotPorts.FullPolicyLoader; +import com.arch.policy.search.domain.snapshot.SnapshotPorts.IncrementalReplayer; +import com.arch.policy.search.infrastructure.demo.DemoSupplierTaskDispatcher; +import com.arch.policy.search.infrastructure.kafka.KafkaPolicyChangeListener; +import com.arch.policy.search.infrastructure.redis.RedisSearchStateStore; +import com.arch.policy.search.infrastructure.redis.SearchFinishedSubscriber; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/architecture/src/main/java/com/arch/policy/MessagePosition.java b/architecture/src/main/java/com/arch/policy/common/model/MessagePosition.java similarity index 95% rename from architecture/src/main/java/com/arch/policy/MessagePosition.java rename to architecture/src/main/java/com/arch/policy/common/model/MessagePosition.java index 50794e1..edca20b 100644 --- a/architecture/src/main/java/com/arch/policy/MessagePosition.java +++ b/architecture/src/main/java/com/arch/policy/common/model/MessagePosition.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.common.model; public final class MessagePosition implements Comparable { public static final MessagePosition BEGINNING = new MessagePosition(0); diff --git a/architecture/src/main/java/com/arch/policy/PolicyChange.java b/architecture/src/main/java/com/arch/policy/common/model/PolicyChange.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/PolicyChange.java rename to architecture/src/main/java/com/arch/policy/common/model/PolicyChange.java index 5dfcecc..b014367 100644 --- a/architecture/src/main/java/com/arch/policy/PolicyChange.java +++ b/architecture/src/main/java/com/arch/policy/common/model/PolicyChange.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.common.model; public final class PolicyChange { public enum Type { UPSERT, DELETE } diff --git a/architecture/src/main/java/com/arch/policy/PolicyRecord.java b/architecture/src/main/java/com/arch/policy/common/model/PolicyRecord.java similarity index 95% rename from architecture/src/main/java/com/arch/policy/PolicyRecord.java rename to architecture/src/main/java/com/arch/policy/common/model/PolicyRecord.java index 42371e3..6fbab70 100644 --- a/architecture/src/main/java/com/arch/policy/PolicyRecord.java +++ b/architecture/src/main/java/com/arch/policy/common/model/PolicyRecord.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.common.model; import java.util.Arrays; import java.util.Collections; diff --git a/architecture/src/main/java/com/arch/policy/api/CallbackResponse.java b/architecture/src/main/java/com/arch/policy/common/search/CallbackResponse.java similarity index 88% rename from architecture/src/main/java/com/arch/policy/api/CallbackResponse.java rename to architecture/src/main/java/com/arch/policy/common/search/CallbackResponse.java index 43b5166..f5fccf7 100644 --- a/architecture/src/main/java/com/arch/policy/api/CallbackResponse.java +++ b/architecture/src/main/java/com/arch/policy/common/search/CallbackResponse.java @@ -1,4 +1,4 @@ -package com.arch.policy.api; +package com.arch.policy.common.search; import java.io.Serializable; diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java b/architecture/src/main/java/com/arch/policy/common/search/PolicySearchRequest.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java rename to architecture/src/main/java/com/arch/policy/common/search/PolicySearchRequest.java index fe9a308..6553673 100644 --- a/architecture/src/main/java/com/arch/policy/api/PolicySearchRequest.java +++ b/architecture/src/main/java/com/arch/policy/common/search/PolicySearchRequest.java @@ -1,4 +1,4 @@ -package com.arch.policy.api; +package com.arch.policy.common.search; import java.io.Serializable; import java.util.ArrayList; diff --git a/architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java b/architecture/src/main/java/com/arch/policy/common/search/PolicySearchResponse.java similarity index 95% rename from architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java rename to architecture/src/main/java/com/arch/policy/common/search/PolicySearchResponse.java index f8b3ecd..fa0d708 100644 --- a/architecture/src/main/java/com/arch/policy/api/PolicySearchResponse.java +++ b/architecture/src/main/java/com/arch/policy/common/search/PolicySearchResponse.java @@ -1,4 +1,4 @@ -package com.arch.policy.api; +package com.arch.policy.common.search; import java.io.Serializable; import java.util.ArrayList; diff --git a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java b/architecture/src/main/java/com/arch/policy/common/search/SupplierCallbackRequest.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java rename to architecture/src/main/java/com/arch/policy/common/search/SupplierCallbackRequest.java index f965f51..a4b5ed2 100644 --- a/architecture/src/main/java/com/arch/policy/api/SupplierCallbackRequest.java +++ b/architecture/src/main/java/com/arch/policy/common/search/SupplierCallbackRequest.java @@ -1,4 +1,4 @@ -package com.arch.policy.api; +package com.arch.policy.common.search; import java.io.Serializable; import java.util.ArrayList; diff --git a/architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java b/architecture/src/main/java/com/arch/policy/search/application/AsyncSearchCoordinator.java similarity index 93% rename from architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java rename to architecture/src/main/java/com/arch/policy/search/application/AsyncSearchCoordinator.java index a58d313..6491af2 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/AsyncSearchCoordinator.java +++ b/architecture/src/main/java/com/arch/policy/search/application/AsyncSearchCoordinator.java @@ -1,8 +1,8 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; -import com.arch.policy.api.PolicySearchRequest; -import com.arch.policy.api.PolicySearchResponse; -import com.arch.policy.api.SupplierCallbackRequest; +import com.arch.policy.common.search.PolicySearchRequest; +import com.arch.policy.common.search.PolicySearchResponse; +import com.arch.policy.common.search.SupplierCallbackRequest; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.ArrayList; diff --git a/architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java b/architecture/src/main/java/com/arch/policy/search/application/LocalSearchWaiters.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java rename to architecture/src/main/java/com/arch/policy/search/application/LocalSearchWaiters.java index 5ccf471..66c27c7 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/LocalSearchWaiters.java +++ b/architecture/src/main/java/com/arch/policy/search/application/LocalSearchWaiters.java @@ -1,4 +1,4 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; diff --git a/architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java b/architecture/src/main/java/com/arch/policy/search/application/PolicyIncrementalUpdater.java similarity index 71% rename from architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java rename to architecture/src/main/java/com/arch/policy/search/application/PolicyIncrementalUpdater.java index d7d1d35..b8deab9 100644 --- a/architecture/src/main/java/com/arch/policy/application/PolicyIncrementalUpdater.java +++ b/architecture/src/main/java/com/arch/policy/search/application/PolicyIncrementalUpdater.java @@ -1,7 +1,7 @@ -package com.arch.policy.application; +package com.arch.policy.search.application; -import com.arch.policy.ActiveSnapshotRegistry; -import com.arch.policy.PolicyChange; +import com.arch.policy.common.model.PolicyChange; +import com.arch.policy.search.domain.snapshot.ActiveSnapshotRegistry; public final class PolicyIncrementalUpdater { private final ActiveSnapshotRegistry registry; diff --git a/architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java b/architecture/src/main/java/com/arch/policy/search/application/PolicyStartupRunner.java similarity index 84% rename from architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java rename to architecture/src/main/java/com/arch/policy/search/application/PolicyStartupRunner.java index 58bbf5f..40ec539 100644 --- a/architecture/src/main/java/com/arch/policy/application/PolicyStartupRunner.java +++ b/architecture/src/main/java/com/arch/policy/search/application/PolicyStartupRunner.java @@ -1,6 +1,6 @@ -package com.arch.policy.application; +package com.arch.policy.search.application; -import com.arch.policy.PolicySnapshotService; +import com.arch.policy.search.domain.snapshot.PolicySnapshotService; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SearchState.java b/architecture/src/main/java/com/arch/policy/search/application/SearchState.java similarity index 57% rename from architecture/src/main/java/com/arch/policy/aggregation/SearchState.java rename to architecture/src/main/java/com/arch/policy/search/application/SearchState.java index fa6ae2e..140010a 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/SearchState.java +++ b/architecture/src/main/java/com/arch/policy/search/application/SearchState.java @@ -1,3 +1,3 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; public enum SearchState { WAITING, COMPLETED, TIMED_OUT } diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java b/architecture/src/main/java/com/arch/policy/search/application/SearchStateStore.java similarity index 91% rename from architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java rename to architecture/src/main/java/com/arch/policy/search/application/SearchStateStore.java index c0a0a33..d4bff02 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/SearchStateStore.java +++ b/architecture/src/main/java/com/arch/policy/search/application/SearchStateStore.java @@ -1,4 +1,4 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; import java.util.List; import java.util.Set; diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java b/architecture/src/main/java/com/arch/policy/search/application/SupplierCallbackService.java similarity index 93% rename from architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java rename to architecture/src/main/java/com/arch/policy/search/application/SupplierCallbackService.java index 6113c36..c3d8c59 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/SupplierCallbackService.java +++ b/architecture/src/main/java/com/arch/policy/search/application/SupplierCallbackService.java @@ -1,6 +1,6 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; -import com.arch.policy.api.SupplierCallbackRequest; +import com.arch.policy.common.search.SupplierCallbackRequest; import com.fasterxml.jackson.databind.ObjectMapper; public final class SupplierCallbackService { diff --git a/architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java b/architecture/src/main/java/com/arch/policy/search/application/SupplierTaskDispatcher.java similarity index 61% rename from architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java rename to architecture/src/main/java/com/arch/policy/search/application/SupplierTaskDispatcher.java index ab43db7..213e734 100644 --- a/architecture/src/main/java/com/arch/policy/aggregation/SupplierTaskDispatcher.java +++ b/architecture/src/main/java/com/arch/policy/search/application/SupplierTaskDispatcher.java @@ -1,6 +1,6 @@ -package com.arch.policy.aggregation; +package com.arch.policy.search.application; -import com.arch.policy.api.PolicySearchRequest; +import com.arch.policy.common.search.PolicySearchRequest; import java.util.Set; diff --git a/architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/ActiveSnapshotRegistry.java similarity index 97% rename from architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/ActiveSnapshotRegistry.java index f086e2c..44cb02b 100644 --- a/architecture/src/main/java/com/arch/policy/ActiveSnapshotRegistry.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/ActiveSnapshotRegistry.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/DefaultSnapshotValidator.java similarity index 76% rename from architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/DefaultSnapshotValidator.java index 2c8c848..2c51ddf 100644 --- a/architecture/src/main/java/com/arch/policy/DefaultSnapshotValidator.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/DefaultSnapshotValidator.java @@ -1,6 +1,8 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; -import static com.arch.policy.SnapshotPorts.SnapshotValidator; +import com.arch.policy.common.model.MessagePosition; + +import static com.arch.policy.search.domain.snapshot.SnapshotPorts.SnapshotValidator; /** Baseline invariants; domain-specific checks can be supplied through SnapshotValidator. */ public final class DefaultSnapshotValidator implements SnapshotValidator { diff --git a/architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/FileSystemSnapshotDirectory.java similarity index 91% rename from architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/FileSystemSnapshotDirectory.java index 64c0bb1..663fed2 100644 --- a/architecture/src/main/java/com/arch/policy/FileSystemSnapshotDirectory.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/FileSystemSnapshotDirectory.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; import java.io.IOException; import java.nio.file.Files; @@ -6,7 +6,7 @@ import java.util.Comparator; import java.util.stream.Stream; -import static com.arch.policy.SnapshotPorts.SnapshotDirectory; +import static com.arch.policy.search.domain.snapshot.SnapshotPorts.SnapshotDirectory; /** Keeps every version in an isolated directory and removes abandoned candidates. */ public final class FileSystemSnapshotDirectory implements SnapshotDirectory { diff --git a/architecture/src/main/java/com/arch/policy/PolicySnapshot.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshot.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/PolicySnapshot.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshot.java index d4e7e53..eb13c1d 100644 --- a/architecture/src/main/java/com/arch/policy/PolicySnapshot.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshot.java @@ -1,4 +1,8 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; + +import com.arch.policy.common.model.MessagePosition; +import com.arch.policy.common.model.PolicyChange; +import com.arch.policy.common.model.PolicyRecord; import org.roaringbitmap.RoaringBitmap; import org.rocksdb.Options; diff --git a/architecture/src/main/java/com/arch/policy/PolicySnapshotService.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshotService.java similarity index 98% rename from architecture/src/main/java/com/arch/policy/PolicySnapshotService.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshotService.java index d90fac8..f2f35f9 100644 --- a/architecture/src/main/java/com/arch/policy/PolicySnapshotService.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/PolicySnapshotService.java @@ -1,4 +1,4 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; diff --git a/architecture/src/main/java/com/arch/policy/SnapshotBuilder.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotBuilder.java similarity index 92% rename from architecture/src/main/java/com/arch/policy/SnapshotBuilder.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotBuilder.java index fbd2ccf..49b8797 100644 --- a/architecture/src/main/java/com/arch/policy/SnapshotBuilder.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotBuilder.java @@ -1,10 +1,12 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; + +import com.arch.policy.common.model.MessagePosition; import org.rocksdb.RocksDBException; import java.nio.file.Path; -import static com.arch.policy.SnapshotPorts.*; +import static com.arch.policy.search.domain.snapshot.SnapshotPorts.*; /** Builds a candidate without exposing it to queries. */ public final class SnapshotBuilder { diff --git a/architecture/src/main/java/com/arch/policy/SnapshotPorts.java b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotPorts.java similarity index 90% rename from architecture/src/main/java/com/arch/policy/SnapshotPorts.java rename to architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotPorts.java index 9e3c4d4..f41f262 100644 --- a/architecture/src/main/java/com/arch/policy/SnapshotPorts.java +++ b/architecture/src/main/java/com/arch/policy/search/domain/snapshot/SnapshotPorts.java @@ -1,4 +1,6 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; + +import com.arch.policy.common.model.MessagePosition; import java.nio.file.Path; diff --git a/architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/demo/DemoSupplierTaskDispatcher.java similarity index 86% rename from architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/demo/DemoSupplierTaskDispatcher.java index ecb3714..7826bf0 100644 --- a/architecture/src/main/java/com/arch/policy/demo/DemoSupplierTaskDispatcher.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/demo/DemoSupplierTaskDispatcher.java @@ -1,9 +1,9 @@ -package com.arch.policy.demo; +package com.arch.policy.search.infrastructure.demo; -import com.arch.policy.aggregation.SupplierCallbackService; -import com.arch.policy.aggregation.SupplierTaskDispatcher; -import com.arch.policy.api.PolicySearchRequest; -import com.arch.policy.api.SupplierCallbackRequest; +import com.arch.policy.search.application.SupplierCallbackService; +import com.arch.policy.search.application.SupplierTaskDispatcher; +import com.arch.policy.common.search.PolicySearchRequest; +import com.arch.policy.common.search.SupplierCallbackRequest; import java.util.Collections; import java.util.Set; diff --git a/architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/KafkaPolicyChangeListener.java similarity index 86% rename from architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/KafkaPolicyChangeListener.java index 595318c..e935aa6 100644 --- a/architecture/src/main/java/com/arch/policy/kafka/KafkaPolicyChangeListener.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/KafkaPolicyChangeListener.java @@ -1,9 +1,9 @@ -package com.arch.policy.kafka; +package com.arch.policy.search.infrastructure.kafka; -import com.arch.policy.MessagePosition; -import com.arch.policy.PolicyChange; -import com.arch.policy.PolicyRecord; -import com.arch.policy.application.PolicyIncrementalUpdater; +import com.arch.policy.common.model.MessagePosition; +import com.arch.policy.common.model.PolicyChange; +import com.arch.policy.common.model.PolicyRecord; +import com.arch.policy.search.application.PolicyIncrementalUpdater; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.kafka.annotation.KafkaListener; diff --git a/architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/PolicyChangeMessage.java similarity index 94% rename from architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/PolicyChangeMessage.java index 3f539ec..9718d5f 100644 --- a/architecture/src/main/java/com/arch/policy/kafka/PolicyChangeMessage.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/kafka/PolicyChangeMessage.java @@ -1,4 +1,4 @@ -package com.arch.policy.kafka; +package com.arch.policy.search.infrastructure.kafka; import java.util.ArrayList; import java.util.List; diff --git a/architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/redis/RedisSearchStateStore.java similarity index 96% rename from architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/redis/RedisSearchStateStore.java index 49547f0..6347e52 100644 --- a/architecture/src/main/java/com/arch/policy/redis/RedisSearchStateStore.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/redis/RedisSearchStateStore.java @@ -1,7 +1,7 @@ -package com.arch.policy.redis; +package com.arch.policy.search.infrastructure.redis; -import com.arch.policy.aggregation.SearchState; -import com.arch.policy.aggregation.SearchStateStore; +import com.arch.policy.search.application.SearchState; +import com.arch.policy.search.application.SearchStateStore; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.script.DefaultRedisScript; diff --git a/architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/redis/SearchFinishedSubscriber.java similarity index 82% rename from architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/redis/SearchFinishedSubscriber.java index f8ea514..208c8f0 100644 --- a/architecture/src/main/java/com/arch/policy/redis/SearchFinishedSubscriber.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/redis/SearchFinishedSubscriber.java @@ -1,6 +1,6 @@ -package com.arch.policy.redis; +package com.arch.policy.search.infrastructure.redis; -import com.arch.policy.aggregation.LocalSearchWaiters; +import com.arch.policy.search.application.LocalSearchWaiters; import org.springframework.data.redis.connection.Message; import org.springframework.data.redis.connection.MessageListener; diff --git a/architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboPolicySearchService.java similarity index 80% rename from architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboPolicySearchService.java index ef51aa6..7fc5cca 100644 --- a/architecture/src/main/java/com/arch/policy/rpc/DubboPolicySearchService.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboPolicySearchService.java @@ -1,9 +1,9 @@ -package com.arch.policy.rpc; +package com.arch.policy.search.infrastructure.rpc; -import com.arch.policy.api.PolicySearchRequest; -import com.arch.policy.api.PolicySearchResponse; -import com.arch.policy.api.PolicySearchRpcService; -import com.arch.policy.aggregation.AsyncSearchCoordinator; +import com.arch.policy.common.search.PolicySearchRequest; +import com.arch.policy.common.search.PolicySearchResponse; +import com.arch.policy.api.search.PolicySearchRpcService; +import com.arch.policy.search.application.AsyncSearchCoordinator; import org.apache.dubbo.config.annotation.DubboService; import org.springframework.beans.factory.annotation.Qualifier; diff --git a/architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java b/architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboSupplierCallbackService.java similarity index 70% rename from architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java rename to architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboSupplierCallbackService.java index f685dc1..ae58dcb 100644 --- a/architecture/src/main/java/com/arch/policy/rpc/DubboSupplierCallbackService.java +++ b/architecture/src/main/java/com/arch/policy/search/infrastructure/rpc/DubboSupplierCallbackService.java @@ -1,9 +1,9 @@ -package com.arch.policy.rpc; +package com.arch.policy.search.infrastructure.rpc; -import com.arch.policy.aggregation.SupplierCallbackService; -import com.arch.policy.api.CallbackResponse; -import com.arch.policy.api.SupplierCallbackRequest; -import com.arch.policy.api.SupplierCallbackRpcService; +import com.arch.policy.search.application.SupplierCallbackService; +import com.arch.policy.common.search.CallbackResponse; +import com.arch.policy.common.search.SupplierCallbackRequest; +import com.arch.policy.api.search.SupplierCallbackRpcService; import org.apache.dubbo.config.annotation.DubboService; @DubboService(version = "1.0.0", timeout = 3000) diff --git a/architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java b/architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java similarity index 95% rename from architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java rename to architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java index fe28969..8117b2a 100644 --- a/architecture/src/test/java/com/arch/policy/AsyncSearchCoordinatorTest.java +++ b/architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java @@ -1,9 +1,8 @@ -package com.arch.policy; +package com.arch.policy.search.application; -import com.arch.policy.aggregation.*; -import com.arch.policy.api.PolicySearchRequest; -import com.arch.policy.api.PolicySearchResponse; -import com.arch.policy.api.SupplierCallbackRequest; +import com.arch.policy.common.search.PolicySearchRequest; +import com.arch.policy.common.search.PolicySearchResponse; +import com.arch.policy.common.search.SupplierCallbackRequest; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; diff --git a/architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java b/architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java similarity index 95% rename from architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java rename to architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java index 297a7c0..9a1ceaf 100644 --- a/architecture/src/test/java/com/arch/policy/SnapshotLifecycleTest.java +++ b/architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java @@ -1,5 +1,7 @@ -package com.arch.policy; +package com.arch.policy.search.domain.snapshot; +import com.arch.policy.common.model.MessagePosition; +import com.arch.policy.common.model.PolicyRecord; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -11,7 +13,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; -import static com.arch.policy.SnapshotPorts.*; +import static com.arch.policy.search.domain.snapshot.SnapshotPorts.*; import static org.junit.jupiter.api.Assertions.*; class SnapshotLifecycleTest { From 4326ef25e694d97b7b9f976a267b360eb071aed7 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Mon, 20 Jul 2026 21:15:38 +0800 Subject: [PATCH 34/37] =?UTF-8?q?=E8=A1=A5=E5=85=85=E4=B8=8B=E5=8D=95?= =?UTF-8?q?=E6=B5=81=E7=A8=8B=E4=BC=AA=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture/README.md | 59 ++++++++- .../policy/api/book/BookOrderRpcService.java | 12 ++ .../BookOrderApplicationService.java | 112 ++++++++++++++++++ .../book/application/BookOrderStore.java | 17 +++ .../book/application/OrderOutboxMessage.java | 30 +++++ .../book/application/OrderStateHistory.java | 33 ++++++ .../application/PostTransitionExecutor.java | 8 ++ .../RetryablePostTransitionExecutor.java | 24 ++++ .../application/TransitionCommitResult.java | 24 ++++ .../arch/policy/book/domain/BookOrder.java | 50 ++++++++ .../book/domain/DefaultOrderTransitions.java | 48 ++++++++ .../arch/policy/book/domain/OrderEvent.java | 15 +++ .../arch/policy/book/domain/OrderState.java | 26 ++++ .../policy/book/domain/OrderStateMachine.java | 72 +++++++++++ .../policy/book/domain/OrderTransition.java | 61 ++++++++++ .../book/domain/RequiredAttributeGuard.java | 16 +++ .../book/domain/StateTransitionContext.java | 30 +++++ .../policy/book/domain/TransitionAction.java | 5 + .../book/domain/TransitionExecution.java | 18 +++ .../policy/book/domain/TransitionGuard.java | 5 + .../repository/InMemoryBookOrderStore.java | 86 ++++++++++++++ .../InMemoryFailedPostActionStore.java | 37 ++++++ .../rpc/DubboBookOrderService.java | 25 ++++ .../policy/common/book/BookOrderRequest.java | 24 ++++ .../policy/common/book/BookOrderResponse.java | 20 ++++ .../policy/common/book/OrderEventRequest.java | 32 +++++ .../common/config/BookOrderConfiguration.java | 36 ++++++ .../BookOrderApplicationServiceTest.java | 105 ++++++++++++++++ .../RetryablePostTransitionExecutorTest.java | 40 +++++++ .../book/domain/OrderStateMachineTest.java | 89 ++++++++++++++ 30 files changed, 1157 insertions(+), 2 deletions(-) create mode 100644 architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderState.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java create mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java create mode 100644 architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java create mode 100644 architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java create mode 100644 architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java create mode 100644 architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java create mode 100644 architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java create mode 100644 architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java create mode 100644 architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java diff --git a/architecture/README.md b/architecture/README.md index 58cde70..04f7058 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -28,7 +28,7 @@ | --- | --- | --- | | 多供应商并发搜索 | 任务分发、异步回调、结果聚合、超时与迟到回调 | 已实现伪代码 | | 海量政策匹配 | 全量加载、增量追平、位图索引、快照校验与热切换 | 已实现伪代码 | -| 交易/下单 | 幂等、状态流转、库存确认、分布式一致性与补偿 | 待补充 | +| 交易/下单 | 幂等下单、受控状态流转、乐观并发与失败隔离 | 已实现核心伪代码 | ## 分层结构 @@ -37,6 +37,7 @@ ```text com.arch.policy ├── api +│ ├── book # 下单及订单状态变更契约 │ └── search # 项目级搜索契约、请求和响应 DTO ├── common │ ├── config # Spring Bean 装配 @@ -50,6 +51,10 @@ com.arch.policy │ ├── kafka # 政策变更消息适配器 │ ├── redis # 聚合状态和完成通知适配器 │ └── rpc # Dubbo 接口实现 +├── book +│ ├── application # 幂等下单、状态变更用例和持久化端口 +│ ├── domain # 订单聚合、状态枚举和合法迁移规则 +│ └── infrastructure # 内存仓储示例和 Dubbo 适配器 └── PolicySearchApplication # 当前搜索场景的启动入口 ``` @@ -146,6 +151,55 @@ flowchart TD Kafka 政策变更消息使用全局单调递增的 `position`,重复或乱序消息会被忽略。如果 Topic 使用多个分区, 生产端必须提供全局序列;否则应将当前位置模型调整为按分区保存 offset。 +## 场景三:幂等下单与订单状态机 + +`BookOrderRpcService.createOrder` 以调用方生成的 `requestId` 作为幂等键。订单通过 `CREATE_SUCCEEDED` 事件 +从 `CREATE` 进入 `WAIT_PAY`。重复请求返回第一次创建的订单,不重复生成业务单。生产实现应在数据库中为 +`request_id` 建唯一索引,并把创单与初始状态流转放入本地事务。 + +外部系统通过 `BookOrderRpcService.fireEvent` 提交支付、出票、验真、取消等业务事件,不能直接指定目标状态。 +`OrderStateMachine` 使用 `(当前状态, 业务事件)` 定位唯一迁移,状态参考 `ipolicytradecore`: + +```mermaid +stateDiagram-v2 + CREATE --> WAIT_PAY: CREATE_SUCCEEDED + CREATE --> CREATE_FAIL: CREATE_FAILED + CREATE --> CANCEL: CANCEL + WAIT_PAY --> BOOKING: PAY_SUCCEEDED + WAIT_PAY --> CANCEL: CANCEL + BOOKING --> BOOKED: BOOK_SUCCEEDED + BOOKING --> BOOK_FAIL: BOOK_FAILED + BOOKING --> VALIDATING: START_VALIDATE + BOOKED --> VALIDATING: START_VALIDATE + BOOKED --> CANCEL: CANCEL + BOOKED --> REFUNDED: REFUND_SUCCEEDED + BOOK_FAIL --> VALIDATING: START_VALIDATE + BOOK_FAIL --> CANCEL: CANCEL + VALIDATING --> BOOKED: VALIDATE_SUCCEEDED + VALIDATING --> VALIDATE_FAIL: VALIDATE_FAILED + VALIDATE_FAIL --> BOOKED: VALIDATE_SUCCEEDED + VALIDATE_FAIL --> CANCEL: CANCEL + CREATE_FAIL --> DELETED: DELETE + BOOK_FAIL --> DELETED: DELETE + CANCEL --> DELETED: DELETE + REFUNDED --> DELETED: DELETE +``` + +一次迁移依次执行 Guard、前置 Action、领域状态变更、原子持久化和后置 Action。`PAY_SUCCEEDED`、 +`BOOK_SUCCEEDED` 等关键事件通过 Guard 校验支付单号、PNR 等业务凭据;业务可以通过迁移 Builder 注册更多 +风控校验、库存检查和任务创建处理器,而不修改状态机引擎。 + +状态事件携带全局唯一 `eventId` 和 `expectedVersion`。仓储在一个事务边界内完成以下写入: + +1. 使用 `order_no + version` compare-and-set 更新订单,阻止并发覆盖。 +2. 保存 `eventId` 处理结果,重复消息直接返回第一次处理的快照。 +3. 追加包含 `from/event/to/operator` 的完整状态历史。 +4. 写入 Outbox 消息,由独立发布任务可靠投递给库存、支付、出票等下游。 + +后置 Action 仅在事务提交后执行,失败会进入 `FailedPostActionStore` 等待重试,不会把已提交订单回滚成旧状态。 +示例使用内存实现展示原子语义;生产落地应使用数据库唯一索引、条件更新、状态历史表、Outbox 表和重试任务, +并由消息消费方继续按照 `eventId` 幂等。 + ## 新增架构场景的约定 新增交易、下单或其他架构伪代码时,应遵循以下约定: @@ -165,4 +219,5 @@ Kafka 政策变更消息使用全局单调递增的 `position`,重复或乱序 mvn -f architecture/pom.xml clean test ``` -当前测试覆盖异步供应商聚合、超时返回部分结果、全量加增量快照构建、运行时快照切换以及失败候选版本隔离。 +当前测试覆盖异步供应商聚合、超时返回部分结果、全量加增量快照构建、运行时快照切换、幂等下单、事件驱动 +状态迁移、Guard/Action 执行顺序、事件幂等、乐观并发、状态历史和 Outbox 原子记录。 diff --git a/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java b/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java new file mode 100644 index 0000000..ff75a2c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java @@ -0,0 +1,12 @@ +package com.arch.policy.api.book; + +import com.arch.policy.common.book.BookOrderRequest; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.book.OrderEventRequest; + +public interface BookOrderRpcService { + + BookOrderResponse createOrder(BookOrderRequest request); + + BookOrderResponse fireEvent(OrderEventRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java new file mode 100644 index 0000000..9531a45 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java @@ -0,0 +1,112 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; +import com.arch.policy.book.domain.OrderEvent; +import com.arch.policy.book.domain.OrderState; +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.domain.StateTransitionContext; +import com.arch.policy.book.domain.TransitionExecution; +import com.arch.policy.common.book.BookOrderRequest; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.book.OrderEventRequest; + +import java.math.BigDecimal; +import java.util.Collections; +import java.util.UUID; + +public final class BookOrderApplicationService { + private final BookOrderStore store; + private final OrderStateMachine stateMachine; + private final PostTransitionExecutor postTransitionExecutor; + + public BookOrderApplicationService(BookOrderStore store, OrderStateMachine stateMachine, + PostTransitionExecutor postTransitionExecutor) { + this.store = store; + this.stateMachine = stateMachine; + this.postTransitionExecutor = postTransitionExecutor; + } + + public BookOrderResponse createOrder(BookOrderRequest request) { + validate(request); + BookOrder existing = store.findByRequestId(request.getRequestId()); + if (existing != null) return response(existing); + + BookOrder order = BookOrder.create(newOrderNo(), request.getRequestId(), request.getCustomerId(), + request.getProductId(), request.getQuantity(), request.getAmount()); + TransitionExecution execution = stateMachine.fire(new StateTransitionContext( + "CREATE:" + request.getRequestId(), OrderEvent.CREATE_SUCCEEDED, order, + request.getCustomerId(), Collections.emptyMap())); + TransitionCommitResult result = store.create(order, history(execution), outbox(execution)); + runPostActionsIfCommitted(result, execution); + return response(result.getOrder()); + } + + public BookOrderResponse fireEvent(OrderEventRequest request) { + validate(request); + BookOrder processed = store.findByEventId(request.getEventId()); + if (processed != null) return response(processed); + BookOrder order = store.findByOrderNo(request.getOrderNo()); + if (order == null) throw new IllegalArgumentException("order not found: " + request.getOrderNo()); + if (order.getVersion() != request.getExpectedVersion()) { + throw new IllegalStateException("stale order version: " + request.getExpectedVersion()); + } + OrderEvent event = parseEvent(request.getEvent()); + TransitionExecution execution = stateMachine.fire(new StateTransitionContext( + request.getEventId(), event, order, request.getOperator(), request.getAttributes())); + TransitionCommitResult result = store.transit(order, execution.getPreviousVersion(), + history(execution), outbox(execution)); + runPostActionsIfCommitted(result, execution); + return response(result.getOrder()); + } + + private void runPostActionsIfCommitted(TransitionCommitResult result, TransitionExecution execution) { + if (!result.isDuplicateEvent()) postTransitionExecutor.execute(stateMachine, execution); + } + + private static OrderStateHistory history(TransitionExecution execution) { + return new OrderStateHistory(execution.getContext().getEventId(), + execution.getContext().getOrder().getOrderNo(), execution.getTransition().getFrom(), + execution.getTransition().getEvent(), execution.getTransition().getTo(), + execution.getContext().getOperator(), System.currentTimeMillis()); + } + + private static OrderOutboxMessage outbox(TransitionExecution execution) { + BookOrder order = execution.getContext().getOrder(); + return new OrderOutboxMessage(execution.getContext().getEventId(), order.getOrderNo(), + execution.getContext().getEvent(), order.getState(), order.getVersion()); + } + + private static void validate(BookOrderRequest request) { + if (request == null) throw new IllegalArgumentException("request is required"); + if (isBlank(request.getRequestId())) throw new IllegalArgumentException("requestId is required"); + if (isBlank(request.getCustomerId())) throw new IllegalArgumentException("customerId is required"); + if (isBlank(request.getProductId())) throw new IllegalArgumentException("productId is required"); + if (request.getQuantity() <= 0) throw new IllegalArgumentException("quantity must be positive"); + if (request.getAmount() == null || request.getAmount().compareTo(BigDecimal.ZERO) < 0) { + throw new IllegalArgumentException("amount must not be negative"); + } + } + + private static void validate(OrderEventRequest request) { + if (request == null || isBlank(request.getEventId()) || isBlank(request.getOrderNo()) + || isBlank(request.getEvent()) || isBlank(request.getOperator())) { + throw new IllegalArgumentException("eventId, orderNo, event and operator are required"); + } + } + + private static OrderEvent parseEvent(String event) { + try { return OrderEvent.valueOf(event.trim().toUpperCase()); } + catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("unknown order event: " + event, failure); + } + } + + private static String newOrderNo() { + return "BO" + UUID.randomUUID().toString().replace("-", "").substring(0, 20).toUpperCase(); + } + + private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } + private static BookOrderResponse response(BookOrder order) { + return new BookOrderResponse(order.getOrderNo(), order.getState().name(), order.getVersion()); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java new file mode 100644 index 0000000..5e69210 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java @@ -0,0 +1,17 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; + +public interface BookOrderStore { + BookOrder findByOrderNo(String orderNo); + + BookOrder findByRequestId(String requestId); + + BookOrder findByEventId(String eventId); + + TransitionCommitResult create(BookOrder order, OrderStateHistory history, + OrderOutboxMessage outboxMessage); + + TransitionCommitResult transit(BookOrder order, long expectedVersion, + OrderStateHistory history, OrderOutboxMessage outboxMessage); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java b/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java new file mode 100644 index 0000000..9e0b4c3 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java @@ -0,0 +1,30 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.OrderEvent; +import com.arch.policy.book.domain.OrderState; + +public final class OrderOutboxMessage { + private final String eventId; + private final String orderNo; + private final OrderEvent event; + private final OrderState state; + private final long orderVersion; + private boolean published; + + public OrderOutboxMessage(String eventId, String orderNo, OrderEvent event, + OrderState state, long orderVersion) { + this.eventId = eventId; + this.orderNo = orderNo; + this.event = event; + this.state = state; + this.orderVersion = orderVersion; + } + + public String getEventId() { return eventId; } + public String getOrderNo() { return orderNo; } + public OrderEvent getEvent() { return event; } + public OrderState getState() { return state; } + public long getOrderVersion() { return orderVersion; } + public boolean isPublished() { return published; } + public void markPublished() { published = true; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java b/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java new file mode 100644 index 0000000..38e8fab --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java @@ -0,0 +1,33 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.OrderEvent; +import com.arch.policy.book.domain.OrderState; + +public final class OrderStateHistory { + private final String eventId; + private final String orderNo; + private final OrderState from; + private final OrderEvent event; + private final OrderState to; + private final String operator; + private final long occurredAtMillis; + + public OrderStateHistory(String eventId, String orderNo, OrderState from, OrderEvent event, + OrderState to, String operator, long occurredAtMillis) { + this.eventId = eventId; + this.orderNo = orderNo; + this.from = from; + this.event = event; + this.to = to; + this.operator = operator; + this.occurredAtMillis = occurredAtMillis; + } + + public String getEventId() { return eventId; } + public String getOrderNo() { return orderNo; } + public OrderState getFrom() { return from; } + public OrderEvent getEvent() { return event; } + public OrderState getTo() { return to; } + public String getOperator() { return operator; } + public long getOccurredAtMillis() { return occurredAtMillis; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java b/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java new file mode 100644 index 0000000..74e5d54 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java @@ -0,0 +1,8 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.domain.TransitionExecution; + +public interface PostTransitionExecutor { + void execute(OrderStateMachine stateMachine, TransitionExecution execution); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java b/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java new file mode 100644 index 0000000..3043b16 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java @@ -0,0 +1,24 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.domain.TransitionExecution; + +public final class RetryablePostTransitionExecutor implements PostTransitionExecutor { + private final FailedPostActionStore failedActionStore; + + public RetryablePostTransitionExecutor(FailedPostActionStore failedActionStore) { + this.failedActionStore = failedActionStore; + } + + @Override public void execute(OrderStateMachine stateMachine, TransitionExecution execution) { + try { + stateMachine.afterCommit(execution); + } catch (RuntimeException failure) { + failedActionStore.record(execution, failure); + } + } + + public interface FailedPostActionStore { + void record(TransitionExecution execution, RuntimeException failure); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java b/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java new file mode 100644 index 0000000..6564e33 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java @@ -0,0 +1,24 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; + +public final class TransitionCommitResult { + private final BookOrder order; + private final boolean duplicateEvent; + + private TransitionCommitResult(BookOrder order, boolean duplicateEvent) { + this.order = order; + this.duplicateEvent = duplicateEvent; + } + + public static TransitionCommitResult committed(BookOrder order) { + return new TransitionCommitResult(order, false); + } + + public static TransitionCommitResult duplicate(BookOrder order) { + return new TransitionCommitResult(order, true); + } + + public BookOrder getOrder() { return order; } + public boolean isDuplicateEvent() { return duplicateEvent; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java b/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java new file mode 100644 index 0000000..3daa8f3 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java @@ -0,0 +1,50 @@ +package com.arch.policy.book.domain; + +import java.math.BigDecimal; + +public final class BookOrder { + private final String orderNo; + private final String requestId; + private final String customerId; + private final String productId; + private final int quantity; + private final BigDecimal amount; + private OrderState state; + private long version; + + private BookOrder(String orderNo, String requestId, String customerId, String productId, + int quantity, BigDecimal amount, OrderState state, long version) { + this.orderNo = orderNo; + this.requestId = requestId; + this.customerId = customerId; + this.productId = productId; + this.quantity = quantity; + this.amount = amount; + this.state = state; + this.version = version; + } + + public static BookOrder create(String orderNo, String requestId, String customerId, + String productId, int quantity, BigDecimal amount) { + return new BookOrder(orderNo, requestId, customerId, productId, quantity, amount, + OrderState.CREATE, 0L); + } + + public BookOrder copy() { + return new BookOrder(orderNo, requestId, customerId, productId, quantity, amount, state, version); + } + + void applyState(OrderState target) { + state = target; + version++; + } + + public String getOrderNo() { return orderNo; } + public String getRequestId() { return requestId; } + public String getCustomerId() { return customerId; } + public String getProductId() { return productId; } + public int getQuantity() { return quantity; } + public BigDecimal getAmount() { return amount; } + public OrderState getState() { return state; } + public long getVersion() { return version; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java b/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java new file mode 100644 index 0000000..bab4e01 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java @@ -0,0 +1,48 @@ +package com.arch.policy.book.domain; + +import java.util.ArrayList; +import java.util.List; + +public final class DefaultOrderTransitions { + private DefaultOrderTransitions() {} + + public static List definitions() { + List transitions = new ArrayList(); + add(transitions, OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY); + add(transitions, OrderState.CREATE, OrderEvent.CREATE_FAILED, OrderState.CREATE_FAIL); + add(transitions, OrderState.CREATE, OrderEvent.CANCEL, OrderState.CANCEL); + transitions.add(OrderTransition + .from(OrderState.WAIT_PAY, OrderEvent.PAY_SUCCEEDED, OrderState.BOOKING) + .guard(new RequiredAttributeGuard("paymentNo")).build()); + add(transitions, OrderState.WAIT_PAY, OrderEvent.CANCEL, OrderState.CANCEL); + transitions.add(OrderTransition + .from(OrderState.BOOKING, OrderEvent.BOOK_SUCCEEDED, OrderState.BOOKED) + .guard(new RequiredAttributeGuard("bookingReference")).build()); + transitions.add(OrderTransition + .from(OrderState.BOOKING, OrderEvent.BOOK_FAILED, OrderState.BOOK_FAIL) + .guard(new RequiredAttributeGuard("failureCode")).build()); + add(transitions, OrderState.BOOKING, OrderEvent.START_VALIDATE, OrderState.VALIDATING); + add(transitions, OrderState.BOOK_FAIL, OrderEvent.START_VALIDATE, OrderState.VALIDATING); + add(transitions, OrderState.BOOK_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); + add(transitions, OrderState.BOOK_FAIL, OrderEvent.DELETE, OrderState.DELETED); + add(transitions, OrderState.BOOKED, OrderEvent.START_VALIDATE, OrderState.VALIDATING); + add(transitions, OrderState.BOOKED, OrderEvent.CANCEL, OrderState.CANCEL); + transitions.add(OrderTransition + .from(OrderState.BOOKED, OrderEvent.REFUND_SUCCEEDED, OrderState.REFUNDED) + .guard(new RequiredAttributeGuard("refundNo")).build()); + add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); + add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_FAILED, OrderState.VALIDATE_FAIL); + add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); + add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); + add(transitions, OrderState.CREATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); + add(transitions, OrderState.CREATE_FAIL, OrderEvent.DELETE, OrderState.DELETED); + add(transitions, OrderState.CANCEL, OrderEvent.DELETE, OrderState.DELETED); + add(transitions, OrderState.REFUNDED, OrderEvent.DELETE, OrderState.DELETED); + return transitions; + } + + private static void add(List transitions, OrderState from, + OrderEvent event, OrderState to) { + transitions.add(OrderTransition.from(from, event, to).build()); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java new file mode 100644 index 0000000..a32a008 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java @@ -0,0 +1,15 @@ +package com.arch.policy.book.domain; + +public enum OrderEvent { + CREATE_SUCCEEDED, + CREATE_FAILED, + PAY_SUCCEEDED, + BOOK_SUCCEEDED, + BOOK_FAILED, + START_VALIDATE, + VALIDATE_SUCCEEDED, + VALIDATE_FAILED, + CANCEL, + REFUND_SUCCEEDED, + DELETE +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java new file mode 100644 index 0000000..a747609 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java @@ -0,0 +1,26 @@ +package com.arch.policy.book.domain; + +public enum OrderState { + CREATE(0, "创单"), + WAIT_PAY(1, "待支付"), + CREATE_FAIL(2, "创单失败"), + CANCEL(3, "已取消"), + BOOKING(4, "预定中"), + BOOKED(5, "已预定"), + BOOK_FAIL(6, "预定失败"), + DELETED(7, "已删除"), + REFUNDED(8, "已退订"), + VALIDATING(10, "验真中"), + VALIDATE_FAIL(11, "验真失败"); + + private final int code; + private final String description; + + OrderState(int code, String description) { + this.code = code; + this.description = description; + } + + public int getCode() { return code; } + public String getDescription() { return description; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java new file mode 100644 index 0000000..7d5b19d --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java @@ -0,0 +1,72 @@ +package com.arch.policy.book.domain; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public final class OrderStateMachine { + private final Map transitions; + + public OrderStateMachine(List definitions) { + Map registry = new HashMap(); + for (OrderTransition transition : definitions) { + TransitionKey key = new TransitionKey(transition.getFrom(), transition.getEvent()); + if (registry.put(key, transition) != null) { + throw new IllegalStateException("duplicate order transition: " + key); + } + } + transitions = Collections.unmodifiableMap(registry); + } + + public TransitionExecution fire(StateTransitionContext context) { + OrderTransition transition = transitions.get( + new TransitionKey(context.getOrder().getState(), context.getEvent())); + if (transition == null) { + throw new IllegalStateException("unsupported order event " + context.getEvent() + + " in state " + context.getOrder().getState()); + } + executeGuards(transition, context); + executeActions(transition.getPreActions(), context); + long previousVersion = context.getOrder().getVersion(); + context.getOrder().applyState(transition.getTo()); + return new TransitionExecution(transition, context, previousVersion); + } + + public void afterCommit(TransitionExecution execution) { + executeActions(execution.getTransition().getPostActions(), execution.getContext()); + } + + public OrderState targetOf(OrderState state, OrderEvent event) { + OrderTransition transition = transitions.get(new TransitionKey(state, event)); + return transition == null ? null : transition.getTo(); + } + + private static void executeGuards(OrderTransition transition, StateTransitionContext context) { + for (TransitionGuard guard : transition.getGuards()) guard.check(context); + } + + private static void executeActions(List actions, StateTransitionContext context) { + for (TransitionAction action : actions) action.execute(context); + } + + private static final class TransitionKey { + private final OrderState state; + private final OrderEvent event; + + private TransitionKey(OrderState state, OrderEvent event) { + this.state = state; + this.event = event; + } + + @Override public boolean equals(Object other) { + if (this == other) return true; + if (!(other instanceof TransitionKey)) return false; + TransitionKey that = (TransitionKey) other; + return state == that.state && event == that.event; + } + + @Override public int hashCode() { return 31 * state.hashCode() + event.hashCode(); } + @Override public String toString() { return state + " + " + event; } + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java new file mode 100644 index 0000000..4ae61df --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java @@ -0,0 +1,61 @@ +package com.arch.policy.book.domain; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class OrderTransition { + private final OrderState from; + private final OrderEvent event; + private final OrderState to; + private final List guards; + private final List preActions; + private final List postActions; + + private OrderTransition(Builder builder) { + from = builder.from; + event = builder.event; + to = builder.to; + guards = immutableCopy(builder.guards); + preActions = immutableCopy(builder.preActions); + postActions = immutableCopy(builder.postActions); + } + + public static Builder from(OrderState from, OrderEvent event, OrderState to) { + return new Builder(from, event, to); + } + + public OrderState getFrom() { return from; } + public OrderEvent getEvent() { return event; } + public OrderState getTo() { return to; } + public List getGuards() { return guards; } + public List getPreActions() { return preActions; } + public List getPostActions() { return postActions; } + + private static List immutableCopy(List values) { + return Collections.unmodifiableList(new ArrayList(values)); + } + + public static final class Builder { + private final OrderState from; + private final OrderEvent event; + private final OrderState to; + private final List guards = new ArrayList(); + private final List preActions = new ArrayList(); + private final List postActions = new ArrayList(); + + private Builder(OrderState from, OrderEvent event, OrderState to) { + if (from == null || event == null || to == null) { + throw new IllegalArgumentException("from, event and to are required"); + } + this.from = from; + this.event = event; + this.to = to; + } + + public Builder guard(TransitionGuard guard) { guards.add(guard); return this; } + public Builder before(TransitionAction action) { preActions.add(action); return this; } + public Builder after(TransitionAction action) { postActions.add(action); return this; } + public OrderTransition build() { return new OrderTransition(this); } + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java b/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java new file mode 100644 index 0000000..b636a00 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java @@ -0,0 +1,16 @@ +package com.arch.policy.book.domain; + +public final class RequiredAttributeGuard implements TransitionGuard { + private final String attributeName; + + public RequiredAttributeGuard(String attributeName) { + this.attributeName = attributeName; + } + + @Override public void check(StateTransitionContext context) { + String value = context.attribute(attributeName); + if (value == null || value.trim().isEmpty()) { + throw new IllegalStateException(attributeName + " is required for event " + context.getEvent()); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java b/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java new file mode 100644 index 0000000..3269260 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java @@ -0,0 +1,30 @@ +package com.arch.policy.book.domain; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public final class StateTransitionContext { + private final String eventId; + private final OrderEvent event; + private final BookOrder order; + private final String operator; + private final Map attributes; + + public StateTransitionContext(String eventId, OrderEvent event, BookOrder order, + String operator, Map attributes) { + this.eventId = eventId; + this.event = event; + this.order = order; + this.operator = operator; + this.attributes = attributes == null ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(attributes)); + } + + public String getEventId() { return eventId; } + public OrderEvent getEvent() { return event; } + public BookOrder getOrder() { return order; } + public String getOperator() { return operator; } + public Map getAttributes() { return attributes; } + public String attribute(String name) { return attributes.get(name); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java new file mode 100644 index 0000000..a035155 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.domain; + +public interface TransitionAction { + void execute(StateTransitionContext context); +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java new file mode 100644 index 0000000..1e4a0c8 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java @@ -0,0 +1,18 @@ +package com.arch.policy.book.domain; + +public final class TransitionExecution { + private final OrderTransition transition; + private final StateTransitionContext context; + private final long previousVersion; + + public TransitionExecution(OrderTransition transition, StateTransitionContext context, + long previousVersion) { + this.transition = transition; + this.context = context; + this.previousVersion = previousVersion; + } + + public OrderTransition getTransition() { return transition; } + public StateTransitionContext getContext() { return context; } + public long getPreviousVersion() { return previousVersion; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java new file mode 100644 index 0000000..5c88ebe --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.domain; + +public interface TransitionGuard { + void check(StateTransitionContext context); +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java new file mode 100644 index 0000000..a76b4af --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java @@ -0,0 +1,86 @@ +package com.arch.policy.book.infrastructure.repository; + +import com.arch.policy.book.application.BookOrderStore; +import com.arch.policy.book.application.OrderOutboxMessage; +import com.arch.policy.book.application.OrderStateHistory; +import com.arch.policy.book.application.TransitionCommitResult; +import com.arch.policy.book.domain.BookOrder; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public final class InMemoryBookOrderStore implements BookOrderStore { + private final Map orders = new HashMap(); + private final Map requestIndexes = new HashMap(); + private final Map eventResults = new HashMap(); + private final List histories = new ArrayList(); + private final List outboxMessages = new ArrayList(); + + @Override public synchronized BookOrder findByOrderNo(String orderNo) { + return copy(orders.get(orderNo)); + } + + @Override public synchronized BookOrder findByRequestId(String requestId) { + String orderNo = requestIndexes.get(requestId); + return orderNo == null ? null : copy(orders.get(orderNo)); + } + + @Override public synchronized BookOrder findByEventId(String eventId) { + return copy(eventResults.get(eventId)); + } + + @Override public synchronized TransitionCommitResult create( + BookOrder order, OrderStateHistory history, OrderOutboxMessage outboxMessage) { + String existingOrderNo = requestIndexes.get(order.getRequestId()); + if (existingOrderNo != null) { + return TransitionCommitResult.duplicate(copy(orders.get(existingOrderNo))); + } + persist(order, history, outboxMessage); + requestIndexes.put(order.getRequestId(), order.getOrderNo()); + return TransitionCommitResult.committed(order.copy()); + } + + @Override public synchronized TransitionCommitResult transit( + BookOrder order, long expectedVersion, OrderStateHistory history, + OrderOutboxMessage outboxMessage) { + BookOrder processed = eventResults.get(history.getEventId()); + if (processed != null) return TransitionCommitResult.duplicate(processed.copy()); + + BookOrder current = orders.get(order.getOrderNo()); + if (current == null || current.getVersion() != expectedVersion + || order.getVersion() != expectedVersion + 1) { + throw new IllegalStateException("concurrent order update: " + order.getOrderNo()); + } + persist(order, history, outboxMessage); + return TransitionCommitResult.committed(order.copy()); + } + + public synchronized List historiesOf(String orderNo) { + List result = new ArrayList(); + for (OrderStateHistory history : histories) { + if (orderNo.equals(history.getOrderNo())) result.add(history); + } + return Collections.unmodifiableList(result); + } + + public synchronized List unpublishedOutboxMessages() { + List result = new ArrayList(); + for (OrderOutboxMessage message : outboxMessages) { + if (!message.isPublished()) result.add(message); + } + return Collections.unmodifiableList(result); + } + + private void persist(BookOrder order, OrderStateHistory history, OrderOutboxMessage outboxMessage) { + BookOrder snapshot = order.copy(); + orders.put(order.getOrderNo(), snapshot); + eventResults.put(history.getEventId(), snapshot.copy()); + histories.add(history); + outboxMessages.add(outboxMessage); + } + + private static BookOrder copy(BookOrder order) { return order == null ? null : order.copy(); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java new file mode 100644 index 0000000..00c7db5 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java @@ -0,0 +1,37 @@ +package com.arch.policy.book.infrastructure.repository; + +import com.arch.policy.book.application.RetryablePostTransitionExecutor.FailedPostActionStore; +import com.arch.policy.book.domain.TransitionExecution; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class InMemoryFailedPostActionStore implements FailedPostActionStore { + private final List failures = new ArrayList(); + + @Override public synchronized void record(TransitionExecution execution, RuntimeException failure) { + failures.add(new FailedPostAction(execution.getContext().getEventId(), + execution.getContext().getOrder().getOrderNo(), failure.getMessage())); + } + + public synchronized List all() { + return Collections.unmodifiableList(new ArrayList(failures)); + } + + public static final class FailedPostAction { + private final String eventId; + private final String orderNo; + private final String reason; + + public FailedPostAction(String eventId, String orderNo, String reason) { + this.eventId = eventId; + this.orderNo = orderNo; + this.reason = reason; + } + + public String getEventId() { return eventId; } + public String getOrderNo() { return orderNo; } + public String getReason() { return reason; } + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java new file mode 100644 index 0000000..12625eb --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java @@ -0,0 +1,25 @@ +package com.arch.policy.book.infrastructure.rpc; + +import com.arch.policy.api.book.BookOrderRpcService; +import com.arch.policy.book.application.BookOrderApplicationService; +import com.arch.policy.common.book.BookOrderRequest; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.book.OrderEventRequest; +import org.apache.dubbo.config.annotation.DubboService; + +@DubboService(version = "1.0.0", timeout = 3000) +public final class DubboBookOrderService implements BookOrderRpcService { + private final BookOrderApplicationService applicationService; + + public DubboBookOrderService(BookOrderApplicationService applicationService) { + this.applicationService = applicationService; + } + + @Override public BookOrderResponse createOrder(BookOrderRequest request) { + return applicationService.createOrder(request); + } + + @Override public BookOrderResponse fireEvent(OrderEventRequest request) { + return applicationService.fireEvent(request); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java b/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java new file mode 100644 index 0000000..d890783 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java @@ -0,0 +1,24 @@ +package com.arch.policy.common.book; + +import java.io.Serializable; +import java.math.BigDecimal; + +public final class BookOrderRequest implements Serializable { + private static final long serialVersionUID = 1L; + private String requestId; + private String customerId; + private String productId; + private int quantity; + private BigDecimal amount; + + public String getRequestId() { return requestId; } + public void setRequestId(String requestId) { this.requestId = requestId; } + public String getCustomerId() { return customerId; } + public void setCustomerId(String customerId) { this.customerId = customerId; } + public String getProductId() { return productId; } + public void setProductId(String productId) { this.productId = productId; } + public int getQuantity() { return quantity; } + public void setQuantity(int quantity) { this.quantity = quantity; } + public BigDecimal getAmount() { return amount; } + public void setAmount(BigDecimal amount) { this.amount = amount; } +} diff --git a/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java b/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java new file mode 100644 index 0000000..4845559 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java @@ -0,0 +1,20 @@ +package com.arch.policy.common.book; + +import java.io.Serializable; + +public final class BookOrderResponse implements Serializable { + private static final long serialVersionUID = 1L; + private final String orderNo; + private final String state; + private final long version; + + public BookOrderResponse(String orderNo, String state, long version) { + this.orderNo = orderNo; + this.state = state; + this.version = version; + } + + public String getOrderNo() { return orderNo; } + public String getState() { return state; } + public long getVersion() { return version; } +} diff --git a/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java b/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java new file mode 100644 index 0000000..0e39d3a --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java @@ -0,0 +1,32 @@ +package com.arch.policy.common.book; + +import java.io.Serializable; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public final class OrderEventRequest implements Serializable { + private static final long serialVersionUID = 1L; + private String eventId; + private String orderNo; + private String event; + private String operator; + private long expectedVersion; + private Map attributes = new HashMap(); + + public String getEventId() { return eventId; } + public void setEventId(String eventId) { this.eventId = eventId; } + public String getOrderNo() { return orderNo; } + public void setOrderNo(String orderNo) { this.orderNo = orderNo; } + public String getEvent() { return event; } + public void setEvent(String event) { this.event = event; } + public String getOperator() { return operator; } + public void setOperator(String operator) { this.operator = operator; } + public long getExpectedVersion() { return expectedVersion; } + public void setExpectedVersion(long expectedVersion) { this.expectedVersion = expectedVersion; } + public Map getAttributes() { return Collections.unmodifiableMap(attributes); } + public void setAttributes(Map attributes) { + this.attributes = attributes == null + ? new HashMap() : new HashMap(attributes); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java new file mode 100644 index 0000000..71716c5 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java @@ -0,0 +1,36 @@ +package com.arch.policy.common.config; + +import com.arch.policy.book.application.BookOrderApplicationService; +import com.arch.policy.book.application.BookOrderStore; +import com.arch.policy.book.application.PostTransitionExecutor; +import com.arch.policy.book.application.RetryablePostTransitionExecutor; +import com.arch.policy.book.domain.DefaultOrderTransitions; +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.infrastructure.repository.InMemoryBookOrderStore; +import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class BookOrderConfiguration { + @Bean public OrderStateMachine orderStateMachine() { + return new OrderStateMachine(DefaultOrderTransitions.definitions()); + } + + @Bean public BookOrderStore bookOrderStore() { return new InMemoryBookOrderStore(); } + + @Bean public InMemoryFailedPostActionStore failedPostActionStore() { + return new InMemoryFailedPostActionStore(); + } + + @Bean public PostTransitionExecutor postTransitionExecutor( + InMemoryFailedPostActionStore failedPostActionStore) { + return new RetryablePostTransitionExecutor(failedPostActionStore); + } + + @Bean public BookOrderApplicationService bookOrderApplicationService( + BookOrderStore store, OrderStateMachine stateMachine, + PostTransitionExecutor postTransitionExecutor) { + return new BookOrderApplicationService(store, stateMachine, postTransitionExecutor); + } +} diff --git a/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java b/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java new file mode 100644 index 0000000..6dffeaf --- /dev/null +++ b/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java @@ -0,0 +1,105 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.DefaultOrderTransitions; +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.infrastructure.repository.InMemoryBookOrderStore; +import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; +import com.arch.policy.common.book.BookOrderRequest; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.book.OrderEventRequest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class BookOrderApplicationServiceTest { + private InMemoryBookOrderStore store; + private BookOrderApplicationService service; + + @BeforeEach + void setUp() { + store = new InMemoryBookOrderStore(); + OrderStateMachine stateMachine = new OrderStateMachine(DefaultOrderTransitions.definitions()); + service = new BookOrderApplicationService(store, stateMachine, + new RetryablePostTransitionExecutor(new InMemoryFailedPostActionStore())); + } + + @Test + void createsOrderIdempotentlyAndCommitsHistoryWithOutbox() { + BookOrderRequest request = request("request-1"); + + BookOrderResponse first = service.createOrder(request); + BookOrderResponse repeated = service.createOrder(request); + + assertEquals("WAIT_PAY", first.getState()); + assertEquals(1L, first.getVersion()); + assertEquals(first.getOrderNo(), repeated.getOrderNo()); + assertEquals(1, store.historiesOf(first.getOrderNo()).size()); + assertEquals(1, store.unpublishedOutboxMessages().size()); + } + + @Test + void drivesOrderByBusinessEventsInsteadOfTargetStates() { + BookOrderResponse created = service.createOrder(request("request-2")); + + BookOrderResponse booking = fire(created, "pay-1", "PAY_SUCCEEDED"); + BookOrderResponse booked = fire(booking, "book-1", "BOOK_SUCCEEDED"); + + assertEquals("BOOKED", booked.getState()); + assertEquals(3L, booked.getVersion()); + assertEquals(3, store.historiesOf(booked.getOrderNo()).size()); + assertEquals(3, store.unpublishedOutboxMessages().size()); + } + + @Test + void returnsOriginalResultForDuplicateEventEvenWithStaleVersion() { + BookOrderResponse created = service.createOrder(request("request-3")); + BookOrderResponse booking = fire(created, "pay-2", "PAY_SUCCEEDED"); + + BookOrderResponse duplicate = fire(created, "pay-2", "PAY_SUCCEEDED"); + + assertEquals(booking.getState(), duplicate.getState()); + assertEquals(booking.getVersion(), duplicate.getVersion()); + assertEquals(2, store.historiesOf(created.getOrderNo()).size()); + } + + @Test + void rejectsUnsupportedEventAndConcurrentUpdate() { + BookOrderResponse created = service.createOrder(request("request-4")); + assertThrows(IllegalStateException.class, + () -> fire(created, "book-too-early", "BOOK_SUCCEEDED")); + + fire(created, "pay-3", "PAY_SUCCEEDED"); + assertThrows(IllegalStateException.class, + () -> fire(created, "cancel-stale", "CANCEL")); + } + + private BookOrderResponse fire(BookOrderResponse current, String eventId, String event) { + OrderEventRequest request = new OrderEventRequest(); + request.setEventId(eventId); + request.setOrderNo(current.getOrderNo()); + request.setExpectedVersion(current.getVersion()); + request.setEvent(event); + request.setOperator("test-operator"); + if ("PAY_SUCCEEDED".equals(event)) { + request.setAttributes(Collections.singletonMap("paymentNo", "payment-1")); + } else if ("BOOK_SUCCEEDED".equals(event)) { + request.setAttributes(Collections.singletonMap("bookingReference", "PNR001")); + } + return service.fireEvent(request); + } + + private static BookOrderRequest request(String requestId) { + BookOrderRequest request = new BookOrderRequest(); + request.setRequestId(requestId); + request.setCustomerId("customer-1"); + request.setProductId("policy-1"); + request.setQuantity(1); + request.setAmount(new BigDecimal("100.00")); + return request; + } +} diff --git a/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java b/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java new file mode 100644 index 0000000..892f71f --- /dev/null +++ b/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java @@ -0,0 +1,40 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; +import com.arch.policy.book.domain.OrderEvent; +import com.arch.policy.book.domain.OrderState; +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.domain.OrderTransition; +import com.arch.policy.book.domain.StateTransitionContext; +import com.arch.policy.book.domain.TransitionExecution; +import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RetryablePostTransitionExecutorTest { + @Test + void recordsPostActionFailureForRetryWithoutRevertingCommittedState() { + OrderTransition transition = OrderTransition + .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) + .after(context -> { throw new IllegalStateException("notification unavailable"); }) + .build(); + OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); + BookOrder order = BookOrder.create("order-1", "request-1", "customer-1", "product-1", + 1, BigDecimal.ONE); + TransitionExecution execution = stateMachine.fire(new StateTransitionContext( + "event-1", OrderEvent.CREATE_SUCCEEDED, order, "tester", + Collections.emptyMap())); + InMemoryFailedPostActionStore failureStore = new InMemoryFailedPostActionStore(); + + new RetryablePostTransitionExecutor(failureStore).execute(stateMachine, execution); + + assertEquals(OrderState.WAIT_PAY, order.getState()); + assertEquals(1, failureStore.all().size()); + assertEquals("event-1", failureStore.all().get(0).getEventId()); + } +} diff --git a/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java b/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java new file mode 100644 index 0000000..9b17b6f --- /dev/null +++ b/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java @@ -0,0 +1,89 @@ +package com.arch.policy.book.domain; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Arrays; +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class OrderStateMachineTest { + @Test + void resolvesTargetByCurrentStateAndBusinessEvent() { + OrderStateMachine stateMachine = new OrderStateMachine(DefaultOrderTransitions.definitions()); + + assertEquals(OrderState.WAIT_PAY, + stateMachine.targetOf(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED)); + assertEquals(OrderState.BOOKING, + stateMachine.targetOf(OrderState.WAIT_PAY, OrderEvent.PAY_SUCCEEDED)); + assertEquals(OrderState.BOOKED, + stateMachine.targetOf(OrderState.VALIDATE_FAIL, OrderEvent.VALIDATE_SUCCEEDED)); + assertNull(stateMachine.targetOf(OrderState.WAIT_PAY, OrderEvent.BOOK_SUCCEEDED)); + } + + @Test + void executesGuardAndPreActionBeforeMutationAndPostActionAfterCommit() { + final AtomicInteger calls = new AtomicInteger(); + TransitionGuard guard = context -> calls.compareAndSet(0, 1); + TransitionAction preAction = context -> { + assertEquals(OrderState.CREATE, context.getOrder().getState()); + calls.compareAndSet(1, 2); + }; + TransitionAction postAction = context -> { + assertEquals(OrderState.WAIT_PAY, context.getOrder().getState()); + calls.compareAndSet(2, 3); + }; + OrderTransition transition = OrderTransition + .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) + .guard(guard).before(preAction).after(postAction).build(); + OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); + BookOrder order = order(); + + TransitionExecution execution = stateMachine.fire(context(order, OrderEvent.CREATE_SUCCEEDED)); + assertEquals(2, calls.get()); + assertEquals(OrderState.WAIT_PAY, order.getState()); + + stateMachine.afterCommit(execution); + assertEquals(3, calls.get()); + } + + @Test + void guardFailurePreventsStateMutation() { + OrderTransition transition = OrderTransition + .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) + .guard(context -> { throw new IllegalStateException("risk rejected"); }) + .build(); + OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); + BookOrder order = order(); + + assertThrows(IllegalStateException.class, + () -> stateMachine.fire(context(order, OrderEvent.CREATE_SUCCEEDED))); + assertEquals(OrderState.CREATE, order.getState()); + assertEquals(0L, order.getVersion()); + } + + @Test + void rejectsDuplicateTransitionRegistration() { + OrderTransition first = OrderTransition + .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY).build(); + OrderTransition duplicate = OrderTransition + .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.CREATE_FAIL).build(); + + assertThrows(IllegalStateException.class, + () -> new OrderStateMachine(Arrays.asList(first, duplicate))); + } + + private static BookOrder order() { + return BookOrder.create("order-1", "request-1", "customer-1", "product-1", + 1, BigDecimal.ONE); + } + + private static StateTransitionContext context(BookOrder order, OrderEvent event) { + return new StateTransitionContext("event-1", event, order, "tester", + Collections.emptyMap()); + } +} From 1bc1bae44018c3b39cfb708b8e093c9d5eee67a3 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Tue, 21 Jul 2026 21:03:02 +0800 Subject: [PATCH 35/37] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=8B=E5=8D=95Seata?= =?UTF-8?q?=20saga=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- architecture/README.md | 90 +++++++-- architecture/pom.xml | 30 +-- .../BookOrderApplicationService.java | 173 +++++++++++------- .../book/application/BookOrderStore.java | 9 +- .../application/CompensationTaskService.java | 51 ++++++ .../application/CreateOrderResultCache.java | 11 ++ .../application/DirectOrderCreationSaga.java | 24 +++ .../application/ExternalResultStatus.java | 7 + .../book/application/GdsBookingGateway.java | 9 + .../book/application/GdsBookingResult.java | 29 +++ .../application/GdsReconciliationService.java | 25 +++ .../book/application/LocalCreateResult.java | 24 +++ .../OrderCreationOutboxMessage.java | 21 +++ .../OrderCreationOutboxPublisher.java | 25 +++ .../book/application/OrderCreationSaga.java | 20 ++ .../OrderCreationSagaStateServices.java | 86 +++++++++ .../OrderEventApplicationService.java | 97 ++++++++++ .../book/application/OrderReconciler.java | 5 + .../book/application/OrderWorkflowTask.java | 19 ++ .../application/OrderWorkflowTaskStore.java | 5 + .../book/application/PaymentGateway.java | 5 + .../application/PromotionStockGateway.java | 5 + .../policy/book/application/RecoveryTask.java | 66 +++++++ .../application/RecoveryTaskProcessor.java | 67 +++++++ .../book/application/RecoveryTaskStore.java | 11 ++ .../arch/policy/book/domain/BookOrder.java | 12 +- .../book/domain/DefaultOrderTransitions.java | 7 + .../arch/policy/book/domain/OrderEvent.java | 2 + .../cache/DefaultCreateOrderResultCache.java | 54 ++++++ .../demo/DemoGdsBookingGateway.java | 18 ++ .../job/OrderCreationOutboxScheduler.java | 15 ++ .../job/OrderRecoveryScheduler.java | 13 ++ .../repository/InMemoryBookOrderStore.java | 35 +++- .../InMemoryOrderWorkflowTaskStore.java | 17 ++ .../repository/InMemoryRecoveryTaskStore.java | 45 +++++ .../rpc/DubboBookOrderService.java | 8 +- .../seata/SeataOrderCreationSaga.java | 37 ++++ .../policy/common/book/BookOrderRequest.java | 3 + .../common/config/BookOrderConfiguration.java | 109 ++++++++++- .../common/config/RedissonConfiguration.java | 28 +++ .../common/config/SeataSagaConfiguration.java | 54 ++++++ .../redis/RedissonDistributedLock.java | 47 +++++ .../redis/RedissonRedisClient.java | 30 +++ .../policy/common/lock/DistributedLock.java | 14 ++ .../DistributedLockUnavailableException.java | 7 + .../arch/policy/common/redis/RedisClient.java | 7 + .../common/redis/RedisClientException.java | 7 + .../src/main/resources/application.yml | 9 + .../main/resources/db/book_order_schema.sql | 37 ++++ .../statelang/book_order_creation_saga.json | 54 ++++++ .../BookOrderApplicationServiceTest.java | 105 ----------- .../RetryablePostTransitionExecutorTest.java | 40 ---- .../book/domain/OrderStateMachineTest.java | 89 --------- .../AsyncSearchCoordinatorTest.java | 98 ---------- .../snapshot/SnapshotLifecycleTest.java | 121 ------------ 55 files changed, 1469 insertions(+), 567 deletions(-) create mode 100644 architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java create mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java create mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java create mode 100644 architecture/src/main/java/com/arch/policy/common/config/RedissonConfiguration.java create mode 100644 architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java create mode 100644 architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonDistributedLock.java create mode 100644 architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonRedisClient.java create mode 100644 architecture/src/main/java/com/arch/policy/common/lock/DistributedLock.java create mode 100644 architecture/src/main/java/com/arch/policy/common/lock/DistributedLockUnavailableException.java create mode 100644 architecture/src/main/java/com/arch/policy/common/redis/RedisClient.java create mode 100644 architecture/src/main/java/com/arch/policy/common/redis/RedisClientException.java create mode 100644 architecture/src/main/resources/db/book_order_schema.sql create mode 100644 architecture/src/main/resources/statelang/book_order_creation_saga.json delete mode 100644 architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java delete mode 100644 architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java delete mode 100644 architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java delete mode 100644 architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java delete mode 100644 architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java diff --git a/architecture/README.md b/architecture/README.md index 04f7058..bc3298a 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -14,9 +14,9 @@ ## 模块目标 - 用最少的代码表达架构中的核心职责、协作关系和一致性约束。 -- 为类似业务问题提供可讨论、可验证、可演进的设计参考。 +- 为类似业务问题提供可讨论、可演进的设计参考。 - 通过接口隔离外部系统,使方案不被 Dubbo、Kafka、Redis 等具体技术绑定。 -- 通过测试验证关键架构行为,而不是覆盖完整的生产业务流程。 +- 通过场景说明和关键注释表达核心架构行为及约束。 - 持续积累搜索、政策匹配、交易、下单等不同场景的方案。 本模块属于架构伪代码,生产落地时仍需根据实际情况补充鉴权、限流、熔断、监控、链路追踪、异常分级、 @@ -153,9 +153,54 @@ Kafka 政策变更消息使用全局单调递增的 `position`,重复或乱序 ## 场景三:幂等下单与订单状态机 -`BookOrderRpcService.createOrder` 以调用方生成的 `requestId` 作为幂等键。订单通过 `CREATE_SUCCEEDED` 事件 -从 `CREATE` 进入 `WAIT_PAY`。重复请求返回第一次创建的订单,不重复生成业务单。生产实现应在数据库中为 -`request_id` 建唯一索引,并把创单与初始状态流转放入本地事务。 +`BookOrderRpcService.createOrder` 以调用方生成的 `requestId` 作为幂等键。下单首先保存 `CREATE` 状态订单, +提交后才由 Seata Saga 调用 GDS,因此不会在本地数据库事务中持有远程调用。 + +```mermaid +sequenceDiagram + autonumber + participant Caller as 调用方 + participant Order as 下单应用服务 + participant DB as 本地数据库 + participant Saga as Seata Saga + participant GDS as GDS + participant Task as 任务/补偿表 + + Caller->>Order: createOrder(requestId) + Order->>Order: 查询 Redis 幂等结果缓存 + Order->>Order: Redisson RLock 获取 requestId 短锁 + Order->>Order: 锁内二次查缓存,仅持锁者查数据库 + Order->>DB: 本地事务写订单 CREATE + DB-->>Order: 提交成功 + Order->>Order: 回填结果缓存并释放 RLock + Order->>DB: 抢占订单创建调度租约 + Order->>Saga: 启动 BookOrderCreationSaga + Saga->>GDS: 发起 PNR 占编 + GDS-->>Saga: SUCCESS / FAIL / UNKNOWN + alt SUCCESS + Saga->>DB: CREATE_SUCCEEDED → WAIT_PAY + Saga->>Task: 幂等创建待支付任务 + else FAIL + Saga->>DB: CREATE_FAILED → CREATE_FAIL + Saga->>Task: 幂等创建促销库存返还任务 + else UNKNOWN + Saga->>DB: START_VALIDATE → VALIDATING + Saga->>Task: 创建 GDS 核对任务 + 人工任务 + end +``` + +订单与 `START_ORDER_CREATION` Outbox 在同一本地事务中提交。事务后立即尝试发布;如果进程在订单提交后、 +启动 Saga 前崩溃,`OrderCreationOutboxScheduler` 会重新投递未发布消息。Saga 使用 `orderNo` 作为业务幂等键, +已经启动的流程由 Seata 根据持久化的状态机日志继续恢复。 +Saga 使用 `orderNo` 作为业务幂等键;GDS 侧也必须使用订单号或稳定请求号保证占编幂等。 + +高并发重复请求首先读取 `book:create:result:{requestId}` 幂等结果缓存,命中后不访问数据库。缓存未命中时, +使用 Redisson `RLock` 锁定 `book:create:lock:{requestId}`,锁内再次检查缓存,只有锁持有者才允许查询数据库和 +执行本地创建事务;未获得锁的请求等待首个请求回填结果缓存,同样不会查询数据库。RLock 最多等待 300ms, +持锁期间由 Redisson watchdog 自动续期;锁只覆盖本地事务,不覆盖耗时不可控的 Saga/GDS 调用。 + +订单状态迁移后会同步刷新结果缓存,缓存 TTL 为 10 分钟。Redis 不可用时锁和缓存主动降级,最终仍由数据库 +`request_id` 唯一索引保证只能创建一张订单;Redisson 锁和结果缓存是防穿透、削峰层,不是最终一致性依据。 外部系统通过 `BookOrderRpcService.fireEvent` 提交支付、出票、验真、取消等业务事件,不能直接指定目标状态。 `OrderStateMachine` 使用 `(当前状态, 业务事件)` 定位唯一迁移,状态参考 `ipolicytradecore`: @@ -164,6 +209,7 @@ Kafka 政策变更消息使用全局单调递增的 `position`,重复或乱序 stateDiagram-v2 CREATE --> WAIT_PAY: CREATE_SUCCEEDED CREATE --> CREATE_FAIL: CREATE_FAILED + CREATE --> VALIDATING: START_VALIDATE CREATE --> CANCEL: CANCEL WAIT_PAY --> BOOKING: PAY_SUCCEEDED WAIT_PAY --> CANCEL: CANCEL @@ -177,6 +223,8 @@ stateDiagram-v2 BOOK_FAIL --> CANCEL: CANCEL VALIDATING --> BOOKED: VALIDATE_SUCCEEDED VALIDATING --> VALIDATE_FAIL: VALIDATE_FAILED + VALIDATING --> WAIT_PAY: GDS_BOOKING_CONFIRMED + VALIDATING --> CREATE_FAIL: GDS_BOOKING_REJECTED VALIDATE_FAIL --> BOOKED: VALIDATE_SUCCEEDED VALIDATE_FAIL --> CANCEL: CANCEL CREATE_FAIL --> DELETED: DELETE @@ -200,6 +248,25 @@ stateDiagram-v2 示例使用内存实现展示原子语义;生产落地应使用数据库唯一索引、条件更新、状态历史表、Outbox 表和重试任务, 并由消息消费方继续按照 `eventId` 幂等。 +### Seata Saga 与补偿 + +`statelang/book_order_creation_saga.json` 是 Seata 状态语言定义,包含 GDS 服务节点、结果 Choice、异常捕获和 +`CancelReservedPnr` 补偿节点。`SeataOrderCreationSaga` 使用 `orderNo` 作为 business key 启动状态机; +`SeataSagaConfiguration` 在存在 `DataSource` 时创建数据库持久化的 `DbStateMachineConfig` 和 +`StateMachineEngine`。没有引擎 Bean 的本地演示环境使用相同 Saga State Services 直接编排,业务行为一致。 + +Saga 提供的是可恢复的最终一致性,而不是把 GDS 变成支持 ACID 回滚的数据库资源。每个外部正向动作都必须有 +幂等补偿语义,并允许空补偿: + +- PNR 占编失败且已经扣减促销库存:创建 `RETURN_PROMOTION_STOCK` 任务。 +- PNR 已占编但采购商取消:创建 `CANCEL_PNR` 任务。 +- 支付成功但最终出票失败:创建 `REFUND_PAYMENT` 任务。 +- GDS 返回 UNKNOWN:创建 `VALIDATE_GDS_BOOKING` 与人工核对任务,定时比较本地状态和 GDS 实际状态。 + +补偿表使用业务唯一键防重;取消 PNR 额外使用 `(order_no, child_order_no, pnr, task_type)` 唯一索引。任务采用 +指数退避重试,超过五次进入 `MANUAL_REQUIRED`。示例 DDL 位于 `db/book_order_schema.sql`;Seata Server 和 +Saga 引擎日志表应使用部署版本对应的官方脚本创建,避免跨版本复制表结构。 + ## 新增架构场景的约定 新增交易、下单或其他架构伪代码时,应遵循以下约定: @@ -208,16 +275,5 @@ stateDiagram-v2 2. 对外契约放在 `api.`,内部实现放在对应的 `` 业务包。 3. 使用 `application/domain/infrastructure` 表达职责边界,不让领域规则依赖具体中间件。 4. 优先提供展示关键协作关系的最小实现,避免把伪代码扩展成不完整的生产框架。 -5. 为幂等、一致性、并发、超时、切换和失败隔离等关键架构行为编写测试。 +5. 用精简的流程代码和注释表达幂等、一致性、并发、超时、切换和失败隔离等关键行为。 6. 在场景文档中记录设计取舍、适用边界,以及生产落地仍需补充的能力。 - -## 运行验证 - -在仓库根目录执行: - -```bash -mvn -f architecture/pom.xml clean test -``` - -当前测试覆盖异步供应商聚合、超时返回部分结果、全量加增量快照构建、运行时快照切换、幂等下单、事件驱动 -状态迁移、Guard/Action 执行顺序、事件幂等、乐观并发、状态历史和 Outbox 原子记录。 diff --git a/architecture/pom.xml b/architecture/pom.xml index 4f7f396..e677418 100644 --- a/architecture/pom.xml +++ b/architecture/pom.xml @@ -12,9 +12,11 @@ 8 8 UTF-8 - 5.10.2 2.7.18 3.2.15 + 1.8.0 + + 3.24.3 @@ -28,17 +30,16 @@ RoaringBitmap 0.9.47 - - org.junit.jupiter - junit-jupiter - ${junit.version} - test - org.springframework.boot spring-boot-starter ${spring-boot.version} + + org.redisson + redisson + ${redisson.version} + org.springframework.boot spring-boot-starter-data-redis @@ -59,15 +60,20 @@ dubbo-spring-boot-starter ${dubbo.version} + + io.seata + seata-saga-engine + ${seata.version} + + + io.seata + seata-saga-engine-store + ${seata.version} + - - org.apache.maven.plugins - maven-surefire-plugin - 3.2.5 - org.springframework.boot spring-boot-maven-plugin diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java index 9531a45..6a9694f 100644 --- a/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java +++ b/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java @@ -1,79 +1,134 @@ package com.arch.policy.book.application; import com.arch.policy.book.domain.BookOrder; -import com.arch.policy.book.domain.OrderEvent; -import com.arch.policy.book.domain.OrderState; -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.domain.StateTransitionContext; -import com.arch.policy.book.domain.TransitionExecution; import com.arch.policy.common.book.BookOrderRequest; import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.book.OrderEventRequest; +import com.arch.policy.common.lock.DistributedLock; +import com.arch.policy.common.lock.DistributedLockUnavailableException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.math.BigDecimal; -import java.util.Collections; import java.util.UUID; public final class BookOrderApplicationService { + private static final Logger LOGGER = LoggerFactory.getLogger(BookOrderApplicationService.class); + private static final DistributedLock.LockHandle NO_OP_LOCK = () -> { }; + private static final long CREATE_LOCK_WAIT_MILLIS = 300L; + private static final String CREATE_ORDER_LOCK_KEY_PREFIX = "book:create:lock:"; + private static final long CREATE_RESULT_WAIT_MILLIS = 500L; + private static final long CREATE_RESULT_TTL_MILLIS = 600_000L; private final BookOrderStore store; - private final OrderStateMachine stateMachine; - private final PostTransitionExecutor postTransitionExecutor; + private final OrderCreationOutboxPublisher outboxPublisher; + private final DistributedLock distributedLock; + private final CreateOrderResultCache resultCache; - public BookOrderApplicationService(BookOrderStore store, OrderStateMachine stateMachine, - PostTransitionExecutor postTransitionExecutor) { + public BookOrderApplicationService(BookOrderStore store, + OrderCreationOutboxPublisher outboxPublisher, + DistributedLock distributedLock, + CreateOrderResultCache resultCache) { this.store = store; - this.stateMachine = stateMachine; - this.postTransitionExecutor = postTransitionExecutor; + this.outboxPublisher = outboxPublisher; + this.distributedLock = distributedLock; + this.resultCache = resultCache; } + /** + * 创建订单并在本地事务提交后启动 GDS 占编 Saga。 + * + *

流程边界:

+ *
    + *
  1. 使用 requestId 查询 Redis 幂等结果缓存,命中则直接返回。
  2. + *
  3. 获取 requestId 对应的 Redisson 短锁并进行缓存二次检查。
  4. + *
  5. 在同一本地事务中保存 CREATE 状态订单和 Saga 启动 Outbox。
  6. + *
  7. 事务提交后发布 Outbox,使用 orderNo 幂等启动 Seata Saga。
  8. + *
  9. Saga 根据 GDS 的 SUCCESS、FAIL、UNKNOWN 结果分别推进至 + * WAIT_PAY、CREATE_FAIL、VALIDATING。
  10. + *
  11. 重新读取订单并返回 Saga 执行后的最新状态。
  12. + *
+ * + *

如果进程在订单提交后、Saga 启动前退出,定时发布器会重新投递未发布的 Outbox; + * Seata 根据持久化的状态机日志恢复已经启动的 Saga。

+ * + * @param request 下单请求,requestId 是业务幂等键 + * @return 当前最新订单状态;同步 Saga 通常返回 WAIT_PAY、CREATE_FAIL 或 VALIDATING + * @throws IllegalArgumentException 请求参数不合法时抛出 + */ public BookOrderResponse createOrder(BookOrderRequest request) { validate(request); - BookOrder existing = store.findByRequestId(request.getRequestId()); - if (existing != null) return response(existing); - - BookOrder order = BookOrder.create(newOrderNo(), request.getRequestId(), request.getCustomerId(), - request.getProductId(), request.getQuantity(), request.getAmount()); - TransitionExecution execution = stateMachine.fire(new StateTransitionContext( - "CREATE:" + request.getRequestId(), OrderEvent.CREATE_SUCCEEDED, order, - request.getCustomerId(), Collections.emptyMap())); - TransitionCommitResult result = store.create(order, history(execution), outbox(execution)); - runPostActionsIfCommitted(result, execution); - return response(result.getOrder()); - } - public BookOrderResponse fireEvent(OrderEventRequest request) { - validate(request); - BookOrder processed = store.findByEventId(request.getEventId()); - if (processed != null) return response(processed); - BookOrder order = store.findByOrderNo(request.getOrderNo()); - if (order == null) throw new IllegalArgumentException("order not found: " + request.getOrderNo()); - if (order.getVersion() != request.getExpectedVersion()) { - throw new IllegalStateException("stale order version: " + request.getExpectedVersion()); + // 锁前只访问 Redis,不查询数据库,避免大量重复请求同时穿透到主库。 + BookOrderResponse cached = resultCache.get(request.getRequestId()); + if (cached != null) return cached; + + DistributedLock.LockHandle lock = acquireCreateOrderLock(request.getRequestId()); + if (lock == null) return responseOfCompetingRequest(request.getRequestId()); + + BookOrder order; + OrderCreationOutboxMessage outboxMessage; + try { + // Double check:前一个持锁请求可能已在等待期间完成创建并回填缓存。 + cached = resultCache.get(request.getRequestId()); + if (cached != null) return cached; + + // 只有锁持有者才能访问数据库;用于缓存丢失、过期及 Redis 降级场景。 + BookOrder existing = store.findByRequestId(request.getRequestId()); + if (existing != null) { + BookOrderResponse existingResponse = response(existing); + resultCache.put(request.getRequestId(), existingResponse, CREATE_RESULT_TTL_MILLIS); + return existingResponse; + } + + // 构造下单对象(模拟下单的内容) + order = BookOrder.create(newOrderNo(), request.getRequestId(), request.getCustomerId(), + request.getProductId(), request.getPromotionId(), request.getQuantity(), + request.getAmount()); + + // 数据库 request_id 唯一索引是最终幂等防线。 + outboxMessage = new OrderCreationOutboxMessage(order.getOrderNo(), + order.getPromotionId()); + LocalCreateResult localResult = store.createOrder(order, outboxMessage); + if (localResult.isDuplicateRequest()) { + BookOrderResponse duplicateResponse = response(localResult.getOrder()); + resultCache.put(request.getRequestId(), duplicateResponse, CREATE_RESULT_TTL_MILLIS); + return duplicateResponse; + } + resultCache.put(request.getRequestId(), response(order), CREATE_RESULT_TTL_MILLIS); + } finally { + lock.close(); } - OrderEvent event = parseEvent(request.getEvent()); - TransitionExecution execution = stateMachine.fire(new StateTransitionContext( - request.getEventId(), event, order, request.getOperator(), request.getAttributes())); - TransitionCommitResult result = store.transit(order, execution.getPreviousVersion(), - history(execution), outbox(execution)); - runPostActionsIfCommitted(result, execution); - return response(result.getOrder()); - } - private void runPostActionsIfCommitted(TransitionCommitResult result, TransitionExecution execution) { - if (!result.isDuplicateEvent()) postTransitionExecutor.execute(stateMachine, execution); + // Redis 锁已释放;立即发布一次,崩溃遗漏由定时 Outbox 发布器补发。 + try { + outboxPublisher.publish(outboxMessage); + } catch (RuntimeException publishFailure) { + LOGGER.warn("Creation Outbox will be retried, orderNo={}", order.getOrderNo(), + publishFailure); + } + + // 同步 Saga 可能已经改变订单状态,因此返回前重新读取聚合快照。 + BookOrder latest = store.findByOrderNo(order.getOrderNo()); + BookOrderResponse latestResponse = response(latest == null ? order : latest); + resultCache.put(request.getRequestId(), latestResponse, CREATE_RESULT_TTL_MILLIS); + return latestResponse; } - private static OrderStateHistory history(TransitionExecution execution) { - return new OrderStateHistory(execution.getContext().getEventId(), - execution.getContext().getOrder().getOrderNo(), execution.getTransition().getFrom(), - execution.getTransition().getEvent(), execution.getTransition().getTo(), - execution.getContext().getOperator(), System.currentTimeMillis()); + private BookOrderResponse responseOfCompetingRequest(String requestId) { + BookOrderResponse cached = resultCache.await(requestId, CREATE_RESULT_WAIT_MILLIS); + if (cached != null) return cached; + throw new IllegalStateException("create order request is being processed: " + requestId); } - private static OrderOutboxMessage outbox(TransitionExecution execution) { - BookOrder order = execution.getContext().getOrder(); - return new OrderOutboxMessage(execution.getContext().getEventId(), order.getOrderNo(), - execution.getContext().getEvent(), order.getState(), order.getVersion()); + private DistributedLock.LockHandle acquireCreateOrderLock(String requestId) { + String key = CREATE_ORDER_LOCK_KEY_PREFIX + requestId; + try { + return distributedLock.tryAcquire(key, CREATE_LOCK_WAIT_MILLIS); + } catch (DistributedLockUnavailableException unavailable) { + // 数据库 request_id 唯一索引仍是最终幂等防线,Redis 故障时允许降级。 + LOGGER.warn("Create-order lock unavailable, fall back to database idempotency, key={}", + key, unavailable); + return NO_OP_LOCK; + } } private static void validate(BookOrderRequest request) { @@ -87,20 +142,6 @@ private static void validate(BookOrderRequest request) { } } - private static void validate(OrderEventRequest request) { - if (request == null || isBlank(request.getEventId()) || isBlank(request.getOrderNo()) - || isBlank(request.getEvent()) || isBlank(request.getOperator())) { - throw new IllegalArgumentException("eventId, orderNo, event and operator are required"); - } - } - - private static OrderEvent parseEvent(String event) { - try { return OrderEvent.valueOf(event.trim().toUpperCase()); } - catch (IllegalArgumentException failure) { - throw new IllegalArgumentException("unknown order event: " + event, failure); - } - } - private static String newOrderNo() { return "BO" + UUID.randomUUID().toString().replace("-", "").substring(0, 20).toUpperCase(); } diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java index 5e69210..cf2bd4e 100644 --- a/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java +++ b/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java @@ -2,6 +2,8 @@ import com.arch.policy.book.domain.BookOrder; +import java.util.List; + public interface BookOrderStore { BookOrder findByOrderNo(String orderNo); @@ -9,8 +11,11 @@ public interface BookOrderStore { BookOrder findByEventId(String eventId); - TransitionCommitResult create(BookOrder order, OrderStateHistory history, - OrderOutboxMessage outboxMessage); + LocalCreateResult createOrder(BookOrder order, OrderCreationOutboxMessage outboxMessage); + + List findUnpublishedCreationOutbox(int limit); + + void markCreationOutboxPublished(String messageId); TransitionCommitResult transit(BookOrder order, long expectedVersion, OrderStateHistory history, OrderOutboxMessage outboxMessage); diff --git a/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java b/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java new file mode 100644 index 0000000..843c8f8 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java @@ -0,0 +1,51 @@ +package com.arch.policy.book.application; + +import java.util.HashMap; +import java.util.Map; + +public final class CompensationTaskService { + private final RecoveryTaskStore taskStore; + + public CompensationTaskService(RecoveryTaskStore taskStore) { this.taskStore = taskStore; } + + public void returnPromotionStock(String orderNo, String promotionId) { + if (isBlank(promotionId)) return; + enqueue("PROMOTION:" + orderNo + ":" + promotionId, + RecoveryTask.RecoveryType.RETURN_PROMOTION_STOCK, orderNo, + params("promotionId", promotionId)); + } + + public void cancelPnr(String orderNo, String childOrderNo, String pnr) { + if (isBlank(pnr)) return; + Map parameters = params("childOrderNo", childOrderNo); + parameters.put("pnr", pnr); + enqueue("CANCEL_PNR:" + orderNo + ":" + childOrderNo + ":" + pnr, + RecoveryTask.RecoveryType.CANCEL_PNR, orderNo, parameters); + } + + public void refundPayment(String orderNo, String paymentNo) { + if (isBlank(paymentNo)) return; + enqueue("REFUND:" + orderNo + ":" + paymentNo, + RecoveryTask.RecoveryType.REFUND_PAYMENT, orderNo, + params("paymentNo", paymentNo)); + } + + public void validateGdsBooking(String orderNo) { + enqueue("VALIDATE_GDS:" + orderNo, + RecoveryTask.RecoveryType.VALIDATE_GDS_BOOKING, orderNo, + new HashMap()); + } + + private void enqueue(String uniqueKey, RecoveryTask.RecoveryType type, String orderNo, + Map parameters) { + taskStore.saveIfAbsent(new RecoveryTask(uniqueKey, type, orderNo, parameters)); + } + + private static Map params(String name, String value) { + Map params = new HashMap(); + params.put(name, value == null ? "" : value); + return params; + } + + private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java b/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java new file mode 100644 index 0000000..db48c56 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java @@ -0,0 +1,11 @@ +package com.arch.policy.book.application; + +import com.arch.policy.common.book.BookOrderResponse; + +public interface CreateOrderResultCache { + BookOrderResponse get(String requestId); + + void put(String requestId, BookOrderResponse response, long ttlMillis); + + BookOrderResponse await(String requestId, long waitMillis); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java new file mode 100644 index 0000000..cda115b --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java @@ -0,0 +1,24 @@ +package com.arch.policy.book.application; + +public final class DirectOrderCreationSaga implements OrderCreationSaga { + private final OrderCreationSagaStateServices services; + + public DirectOrderCreationSaga(OrderCreationSagaStateServices services) { this.services = services; } + + @Override public SagaStartResult start(String orderNo, String promotionId) { + GdsBookingResult result; + try { + result = services.reservePnr(orderNo); + } catch (RuntimeException uncertainFailure) { + result = GdsBookingResult.unknown(uncertainFailure.getClass().getSimpleName()); + } + if (result.getStatus() == ExternalResultStatus.SUCCESS) { + services.markCreateSuccess(orderNo, result.getPnr()); + } else if (result.getStatus() == ExternalResultStatus.FAIL) { + services.markCreateFailure(orderNo, result.getErrorCode(), promotionId); + } else { + services.markCreateUnknown(orderNo, result.getErrorCode()); + } + return new SagaStartResult("DIRECT:" + orderNo, false); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java b/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java new file mode 100644 index 0000000..099da1a --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java @@ -0,0 +1,7 @@ +package com.arch.policy.book.application; + +public enum ExternalResultStatus { + SUCCESS, + FAIL, + UNKNOWN +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java new file mode 100644 index 0000000..cc689f1 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java @@ -0,0 +1,9 @@ +package com.arch.policy.book.application; + +public interface GdsBookingGateway { + GdsBookingResult createPnr(String orderNo); + + GdsBookingResult queryPnr(String orderNo); + + void cancelPnr(String orderNo, String childOrderNo, String pnr); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java new file mode 100644 index 0000000..72b6d20 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java @@ -0,0 +1,29 @@ +package com.arch.policy.book.application; + +public final class GdsBookingResult { + private final ExternalResultStatus status; + private final String pnr; + private final String errorCode; + + public GdsBookingResult(ExternalResultStatus status, String pnr, String errorCode) { + this.status = status; + this.pnr = pnr; + this.errorCode = errorCode; + } + + public static GdsBookingResult success(String pnr) { + return new GdsBookingResult(ExternalResultStatus.SUCCESS, pnr, null); + } + + public static GdsBookingResult fail(String errorCode) { + return new GdsBookingResult(ExternalResultStatus.FAIL, null, errorCode); + } + + public static GdsBookingResult unknown(String errorCode) { + return new GdsBookingResult(ExternalResultStatus.UNKNOWN, null, errorCode); + } + + public ExternalResultStatus getStatus() { return status; } + public String getPnr() { return pnr; } + public String getErrorCode() { return errorCode; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java b/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java new file mode 100644 index 0000000..8527498 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java @@ -0,0 +1,25 @@ +package com.arch.policy.book.application; + +public final class GdsReconciliationService implements OrderReconciler { + private final GdsBookingGateway gdsGateway; + private final OrderCreationSagaStateServices sagaStates; + + public GdsReconciliationService(GdsBookingGateway gdsGateway, + OrderCreationSagaStateServices sagaStates) { + this.gdsGateway = gdsGateway; + this.sagaStates = sagaStates; + } + + @Override public void reconcile(String orderNo) { + GdsBookingResult result = gdsGateway.queryPnr(orderNo); + if (result.getStatus() == ExternalResultStatus.SUCCESS) { + sagaStates.markGdsValidationSuccess(orderNo, result.getPnr()); + return; + } + if (result.getStatus() == ExternalResultStatus.FAIL) { + sagaStates.markGdsValidationFailure(orderNo, result.getErrorCode()); + return; + } + throw new IllegalStateException("GDS result is still unknown"); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java b/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java new file mode 100644 index 0000000..dbeaadd --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java @@ -0,0 +1,24 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; + +public final class LocalCreateResult { + private final BookOrder order; + private final boolean duplicateRequest; + + private LocalCreateResult(BookOrder order, boolean duplicateRequest) { + this.order = order; + this.duplicateRequest = duplicateRequest; + } + + public static LocalCreateResult created(BookOrder order) { + return new LocalCreateResult(order, false); + } + + public static LocalCreateResult duplicate(BookOrder order) { + return new LocalCreateResult(order, true); + } + + public BookOrder getOrder() { return order; } + public boolean isDuplicateRequest() { return duplicateRequest; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java new file mode 100644 index 0000000..cd52227 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java @@ -0,0 +1,21 @@ +package com.arch.policy.book.application; + +/** 与订单在同一本地事务中保存,用于可靠启动创建 Saga。 */ +public final class OrderCreationOutboxMessage { + private final String messageId; + private final String orderNo; + private final String promotionId; + private boolean published; + + public OrderCreationOutboxMessage(String orderNo, String promotionId) { + this.messageId = "START_ORDER_CREATION:" + orderNo; + this.orderNo = orderNo; + this.promotionId = promotionId; + } + + public String getMessageId() { return messageId; } + public String getOrderNo() { return orderNo; } + public String getPromotionId() { return promotionId; } + public boolean isPublished() { return published; } + public void markPublished() { published = true; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java new file mode 100644 index 0000000..a1c5ec7 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java @@ -0,0 +1,25 @@ +package com.arch.policy.book.application; + +import java.util.List; + +/** 以至少一次语义发布 Outbox;OrderCreationSaga 必须使用 orderNo 幂等启动。 */ +public final class OrderCreationOutboxPublisher { + private final BookOrderStore store; + private final OrderCreationSaga creationSaga; + + public OrderCreationOutboxPublisher(BookOrderStore store, OrderCreationSaga creationSaga) { + this.store = store; + this.creationSaga = creationSaga; + } + + public void publish(OrderCreationOutboxMessage message) { + if (message.isPublished()) return; + creationSaga.start(message.getOrderNo(), message.getPromotionId()); + store.markCreationOutboxPublished(message.getMessageId()); + } + + public void publishPending(int limit) { + List messages = store.findUnpublishedCreationOutbox(limit); + for (OrderCreationOutboxMessage message : messages) publish(message); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java new file mode 100644 index 0000000..9f9637c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java @@ -0,0 +1,20 @@ +package com.arch.policy.book.application; + +public interface OrderCreationSaga { + + /** 使用 orderNo 作为业务键幂等启动;重复调用不得重复执行外部业务动作。 */ + SagaStartResult start(String orderNo, String promotionId); + + final class SagaStartResult { + private final String instanceId; + private final boolean running; + + public SagaStartResult(String instanceId, boolean running) { + this.instanceId = instanceId; + this.running = running; + } + + public String getInstanceId() { return instanceId; } + public boolean isRunning() { return running; } + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java new file mode 100644 index 0000000..366c613 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java @@ -0,0 +1,86 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; +import com.arch.policy.common.book.OrderEventRequest; + +import java.util.Collections; + +public final class OrderCreationSagaStateServices { + private final BookOrderStore store; + private final GdsBookingGateway gdsGateway; + private final OrderEventApplicationService eventService; + private final CompensationTaskService compensationService; + private final OrderWorkflowTaskStore workflowTaskStore; + + public OrderCreationSagaStateServices(BookOrderStore store, GdsBookingGateway gdsGateway, + OrderEventApplicationService eventService, + CompensationTaskService compensationService, + OrderWorkflowTaskStore workflowTaskStore) { + this.store = store; + this.gdsGateway = gdsGateway; + this.eventService = eventService; + this.compensationService = compensationService; + this.workflowTaskStore = workflowTaskStore; + } + + public GdsBookingResult reservePnr(String orderNo) { return gdsGateway.createPnr(orderNo); } + + public boolean markCreateSuccess(String orderNo, String pnr) { + fire(orderNo, orderNo + ":CREATE_SUCCESS", "CREATE_SUCCEEDED", "gds-booking-saga", + Collections.singletonMap("bookingReference", value(pnr))); + workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("WAIT_PAYMENT:" + orderNo, + orderNo, OrderWorkflowTask.TaskType.WAIT_PAYMENT)); + return true; + } + + public boolean markCreateFailure(String orderNo, String errorCode, String promotionId) { + fire(orderNo, orderNo + ":CREATE_FAIL", "CREATE_FAILED", "gds-booking-saga", + Collections.singletonMap("failureCode", value(errorCode))); + compensationService.returnPromotionStock(orderNo, value(promotionId)); + return true; + } + + public boolean markCreateUnknown(String orderNo, String errorCode) { + fire(orderNo, orderNo + ":CREATE_UNKNOWN", "START_VALIDATE", "gds-booking-saga", + Collections.singletonMap("failureCode", value(errorCode))); + workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("MANUAL_GDS_VALIDATION:" + orderNo, + orderNo, OrderWorkflowTask.TaskType.MANUAL_GDS_VALIDATION)); + compensationService.validateGdsBooking(orderNo); + return true; + } + + public boolean enqueueCancelPnr(String orderNo, String childOrderNo, String pnr) { + compensationService.cancelPnr(orderNo, childOrderNo, pnr); + return true; + } + + public void markGdsValidationSuccess(String orderNo, String pnr) { + fire(orderNo, orderNo + ":RECONCILE_SUCCESS", "GDS_BOOKING_CONFIRMED", + "gds-reconciliation-job", Collections.singletonMap("bookingReference", value(pnr))); + workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("WAIT_PAYMENT:" + orderNo, + orderNo, OrderWorkflowTask.TaskType.WAIT_PAYMENT)); + } + + public void markGdsValidationFailure(String orderNo, String errorCode) { + BookOrder order = store.findByOrderNo(orderNo); + fire(orderNo, orderNo + ":RECONCILE_FAIL", "GDS_BOOKING_REJECTED", + "gds-reconciliation-job", Collections.singletonMap("failureCode", value(errorCode))); + compensationService.returnPromotionStock(orderNo, value(order.getPromotionId())); + } + + private void fire(String orderNo, String eventId, String event, String operator, + java.util.Map attributes) { + BookOrder order = store.findByOrderNo(orderNo); + OrderEventRequest request = new OrderEventRequest(); + request.setOrderNo(orderNo); + request.setEventId(eventId); + request.setEvent(event); + request.setOperator(operator); + request.setExpectedVersion(order.getVersion()); + request.setAttributes(attributes); + eventService.fireEvent(request); + } + + private static String value(String value) { return value == null ? "" : value; } + +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java b/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java new file mode 100644 index 0000000..4dc9c52 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java @@ -0,0 +1,97 @@ +package com.arch.policy.book.application; + +import com.arch.policy.book.domain.BookOrder; +import com.arch.policy.book.domain.OrderEvent; +import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.domain.StateTransitionContext; +import com.arch.policy.book.domain.TransitionExecution; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.book.OrderEventRequest; + +public final class OrderEventApplicationService { + private static final long CREATE_RESULT_TTL_MILLIS = 600_000L; + private final BookOrderStore store; + private final OrderStateMachine stateMachine; + private final PostTransitionExecutor postTransitionExecutor; + private final CompensationTaskService compensationTaskService; + private final CreateOrderResultCache resultCache; + + public OrderEventApplicationService(BookOrderStore store, OrderStateMachine stateMachine, + PostTransitionExecutor postTransitionExecutor, + CompensationTaskService compensationTaskService, + CreateOrderResultCache resultCache) { + this.store = store; + this.stateMachine = stateMachine; + this.postTransitionExecutor = postTransitionExecutor; + this.compensationTaskService = compensationTaskService; + this.resultCache = resultCache; + } + + public BookOrderResponse fireEvent(OrderEventRequest request) { + validate(request); + BookOrder processed = store.findByEventId(request.getEventId()); + if (processed != null) return response(processed); + BookOrder order = store.findByOrderNo(request.getOrderNo()); + if (order == null) throw new IllegalArgumentException("order not found: " + request.getOrderNo()); + if (order.getVersion() != request.getExpectedVersion()) { + throw new IllegalStateException("stale order version: " + request.getExpectedVersion()); + } + OrderEvent event = parseEvent(request.getEvent()); + TransitionExecution execution = stateMachine.fire(new StateTransitionContext( + request.getEventId(), event, order, request.getOperator(), request.getAttributes())); + TransitionCommitResult result = store.transit(order, execution.getPreviousVersion(), + history(execution), outbox(execution)); + if (!result.isDuplicateEvent()) { + enqueueCompensation(execution); + postTransitionExecutor.execute(stateMachine, execution); + } + BookOrderResponse response = response(result.getOrder()); + resultCache.put(result.getOrder().getRequestId(), response, CREATE_RESULT_TTL_MILLIS); + return response; + } + + private void enqueueCompensation(TransitionExecution execution) { + StateTransitionContext context = execution.getContext(); + if (context.getEvent() == OrderEvent.CANCEL && !isBlank(context.attribute("pnr"))) { + compensationTaskService.cancelPnr(context.getOrder().getOrderNo(), + value(context.attribute("childOrderNo")), context.attribute("pnr")); + } + if (context.getEvent() == OrderEvent.BOOK_FAILED && !isBlank(context.attribute("paymentNo"))) { + compensationTaskService.refundPayment(context.getOrder().getOrderNo(), + context.attribute("paymentNo")); + } + } + + private static OrderStateHistory history(TransitionExecution execution) { + return new OrderStateHistory(execution.getContext().getEventId(), + execution.getContext().getOrder().getOrderNo(), execution.getTransition().getFrom(), + execution.getTransition().getEvent(), execution.getTransition().getTo(), + execution.getContext().getOperator(), System.currentTimeMillis()); + } + + private static OrderOutboxMessage outbox(TransitionExecution execution) { + BookOrder order = execution.getContext().getOrder(); + return new OrderOutboxMessage(execution.getContext().getEventId(), order.getOrderNo(), + execution.getContext().getEvent(), order.getState(), order.getVersion()); + } + + private static void validate(OrderEventRequest request) { + if (request == null || isBlank(request.getEventId()) || isBlank(request.getOrderNo()) + || isBlank(request.getEvent()) || isBlank(request.getOperator())) { + throw new IllegalArgumentException("eventId, orderNo, event and operator are required"); + } + } + + private static OrderEvent parseEvent(String event) { + try { return OrderEvent.valueOf(event.trim().toUpperCase()); } + catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("unknown order event: " + event, failure); + } + } + + private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } + private static String value(String value) { return value == null ? "" : value; } + private static BookOrderResponse response(BookOrder order) { + return new BookOrderResponse(order.getOrderNo(), order.getState().name(), order.getVersion()); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java b/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java new file mode 100644 index 0000000..2420f70 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.application; + +public interface OrderReconciler { + void reconcile(String orderNo); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java new file mode 100644 index 0000000..de6d653 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java @@ -0,0 +1,19 @@ +package com.arch.policy.book.application; + +public final class OrderWorkflowTask { + private final String uniqueKey; + private final String orderNo; + private final TaskType type; + + public OrderWorkflowTask(String uniqueKey, String orderNo, TaskType type) { + this.uniqueKey = uniqueKey; + this.orderNo = orderNo; + this.type = type; + } + + public String getUniqueKey() { return uniqueKey; } + public String getOrderNo() { return orderNo; } + public TaskType getType() { return type; } + + public enum TaskType { WAIT_PAYMENT, MANUAL_GDS_VALIDATION } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java new file mode 100644 index 0000000..88fb2d5 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.application; + +public interface OrderWorkflowTaskStore { + void saveIfAbsent(OrderWorkflowTask task); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java b/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java new file mode 100644 index 0000000..6c828b2 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.application; + +public interface PaymentGateway { + void refund(String orderNo, String paymentNo); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java b/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java new file mode 100644 index 0000000..23a9132 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java @@ -0,0 +1,5 @@ +package com.arch.policy.book.application; + +public interface PromotionStockGateway { + void returnStock(String orderNo, String promotionId); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java new file mode 100644 index 0000000..0989144 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java @@ -0,0 +1,66 @@ +package com.arch.policy.book.application; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public final class RecoveryTask { + private final String uniqueKey; + private final RecoveryType type; + private final String orderNo; + private final Map parameters; + private RecoveryStatus status; + private int attempts; + private long nextAttemptAtMillis; + private String lastError; + + public RecoveryTask(String uniqueKey, RecoveryType type, String orderNo, + Map parameters) { + this.uniqueKey = uniqueKey; + this.type = type; + this.orderNo = orderNo; + this.parameters = parameters == null ? Collections.emptyMap() + : Collections.unmodifiableMap(new HashMap(parameters)); + this.status = RecoveryStatus.PENDING; + } + + public RecoveryTask copy() { + RecoveryTask copy = new RecoveryTask(uniqueKey, type, orderNo, parameters); + copy.status = status; + copy.attempts = attempts; + copy.nextAttemptAtMillis = nextAttemptAtMillis; + copy.lastError = lastError; + return copy; + } + + public void markSucceeded() { status = RecoveryStatus.SUCCEEDED; lastError = null; } + public void markRetry(long nextAttemptAtMillis, String error) { + attempts++; + status = RecoveryStatus.RETRYING; + this.nextAttemptAtMillis = nextAttemptAtMillis; + lastError = error; + } + public void markManualRequired(String error) { + attempts++; + status = RecoveryStatus.MANUAL_REQUIRED; + lastError = error; + } + + public String getUniqueKey() { return uniqueKey; } + public RecoveryType getType() { return type; } + public String getOrderNo() { return orderNo; } + public Map getParameters() { return parameters; } + public RecoveryStatus getStatus() { return status; } + public int getAttempts() { return attempts; } + public long getNextAttemptAtMillis() { return nextAttemptAtMillis; } + public String getLastError() { return lastError; } + + public enum RecoveryType { + RETURN_PROMOTION_STOCK, + CANCEL_PNR, + REFUND_PAYMENT, + VALIDATE_GDS_BOOKING + } + + public enum RecoveryStatus { PENDING, RETRYING, SUCCEEDED, MANUAL_REQUIRED } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java new file mode 100644 index 0000000..c010fda --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java @@ -0,0 +1,67 @@ +package com.arch.policy.book.application; + +import java.util.List; + +public final class RecoveryTaskProcessor { + private static final int MAX_ATTEMPTS = 5; + private final RecoveryTaskStore taskStore; + private final PromotionStockGateway promotionStockGateway; + private final GdsBookingGateway gdsGateway; + private final PaymentGateway paymentGateway; + private final OrderReconciler reconciliationService; + + public RecoveryTaskProcessor(RecoveryTaskStore taskStore, + PromotionStockGateway promotionStockGateway, + GdsBookingGateway gdsGateway, + PaymentGateway paymentGateway, + OrderReconciler reconciliationService) { + this.taskStore = taskStore; + this.promotionStockGateway = promotionStockGateway; + this.gdsGateway = gdsGateway; + this.paymentGateway = paymentGateway; + this.reconciliationService = reconciliationService; + } + + public void processBatch(long nowMillis, int limit) { + List tasks = taskStore.findExecutable(nowMillis, limit); + for (RecoveryTask task : tasks) process(task, nowMillis); + } + + private void process(RecoveryTask task, long nowMillis) { + try { + execute(task); + task.markSucceeded(); + } catch (RuntimeException failure) { + if (task.getAttempts() + 1 >= MAX_ATTEMPTS) { + task.markManualRequired(failure.getMessage()); + } else { + task.markRetry(nowMillis + retryDelayMillis(task.getAttempts()), failure.getMessage()); + } + } + taskStore.save(task); + } + + private void execute(RecoveryTask task) { + switch (task.getType()) { + case RETURN_PROMOTION_STOCK: + promotionStockGateway.returnStock(task.getOrderNo(), task.getParameters().get("promotionId")); + return; + case CANCEL_PNR: + gdsGateway.cancelPnr(task.getOrderNo(), task.getParameters().get("childOrderNo"), + task.getParameters().get("pnr")); + return; + case REFUND_PAYMENT: + paymentGateway.refund(task.getOrderNo(), task.getParameters().get("paymentNo")); + return; + case VALIDATE_GDS_BOOKING: + reconciliationService.reconcile(task.getOrderNo()); + return; + default: + throw new IllegalStateException("unsupported recovery type: " + task.getType()); + } + } + + private static long retryDelayMillis(int attempts) { + return Math.min(60_000L, 1_000L << Math.min(attempts, 6)); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java new file mode 100644 index 0000000..b8ac806 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java @@ -0,0 +1,11 @@ +package com.arch.policy.book.application; + +import java.util.List; + +public interface RecoveryTaskStore { + RecoveryTask saveIfAbsent(RecoveryTask task); + + List findExecutable(long nowMillis, int limit); + + void save(RecoveryTask task); +} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java b/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java index 3daa8f3..6507001 100644 --- a/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java +++ b/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java @@ -7,17 +7,19 @@ public final class BookOrder { private final String requestId; private final String customerId; private final String productId; + private final String promotionId; private final int quantity; private final BigDecimal amount; private OrderState state; private long version; private BookOrder(String orderNo, String requestId, String customerId, String productId, - int quantity, BigDecimal amount, OrderState state, long version) { + String promotionId, int quantity, BigDecimal amount, OrderState state, long version) { this.orderNo = orderNo; this.requestId = requestId; this.customerId = customerId; this.productId = productId; + this.promotionId = promotionId; this.quantity = quantity; this.amount = amount; this.state = state; @@ -25,13 +27,14 @@ private BookOrder(String orderNo, String requestId, String customerId, String pr } public static BookOrder create(String orderNo, String requestId, String customerId, - String productId, int quantity, BigDecimal amount) { - return new BookOrder(orderNo, requestId, customerId, productId, quantity, amount, + String productId, String promotionId, int quantity, BigDecimal amount) { + return new BookOrder(orderNo, requestId, customerId, productId, promotionId, quantity, amount, OrderState.CREATE, 0L); } public BookOrder copy() { - return new BookOrder(orderNo, requestId, customerId, productId, quantity, amount, state, version); + return new BookOrder(orderNo, requestId, customerId, productId, promotionId, + quantity, amount, state, version); } void applyState(OrderState target) { @@ -43,6 +46,7 @@ void applyState(OrderState target) { public String getRequestId() { return requestId; } public String getCustomerId() { return customerId; } public String getProductId() { return productId; } + public String getPromotionId() { return promotionId; } public int getQuantity() { return quantity; } public BigDecimal getAmount() { return amount; } public OrderState getState() { return state; } diff --git a/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java b/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java index bab4e01..2d51f92 100644 --- a/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java +++ b/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java @@ -10,6 +10,7 @@ public static List definitions() { List transitions = new ArrayList(); add(transitions, OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY); add(transitions, OrderState.CREATE, OrderEvent.CREATE_FAILED, OrderState.CREATE_FAIL); + add(transitions, OrderState.CREATE, OrderEvent.START_VALIDATE, OrderState.VALIDATING); add(transitions, OrderState.CREATE, OrderEvent.CANCEL, OrderState.CANCEL); transitions.add(OrderTransition .from(OrderState.WAIT_PAY, OrderEvent.PAY_SUCCEEDED, OrderState.BOOKING) @@ -32,6 +33,12 @@ public static List definitions() { .guard(new RequiredAttributeGuard("refundNo")).build()); add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_FAILED, OrderState.VALIDATE_FAIL); + transitions.add(OrderTransition + .from(OrderState.VALIDATING, OrderEvent.GDS_BOOKING_CONFIRMED, OrderState.WAIT_PAY) + .guard(new RequiredAttributeGuard("bookingReference")).build()); + transitions.add(OrderTransition + .from(OrderState.VALIDATING, OrderEvent.GDS_BOOKING_REJECTED, OrderState.CREATE_FAIL) + .guard(new RequiredAttributeGuard("failureCode")).build()); add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); add(transitions, OrderState.CREATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java index a32a008..9bd36b7 100644 --- a/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java +++ b/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java @@ -9,6 +9,8 @@ public enum OrderEvent { START_VALIDATE, VALIDATE_SUCCEEDED, VALIDATE_FAILED, + GDS_BOOKING_CONFIRMED, + GDS_BOOKING_REJECTED, CANCEL, REFUND_SUCCEEDED, DELETE diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java new file mode 100644 index 0000000..fd6bf92 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java @@ -0,0 +1,54 @@ +package com.arch.policy.book.infrastructure.cache; + +import com.arch.policy.book.application.CreateOrderResultCache; +import com.arch.policy.common.book.BookOrderResponse; +import com.arch.policy.common.redis.RedisClient; +import com.arch.policy.common.redis.RedisClientException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.locks.LockSupport; + +public final class DefaultCreateOrderResultCache implements CreateOrderResultCache { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultCreateOrderResultCache.class); + private static final String KEY_PREFIX = "book:create:result:"; + private static final long POLL_INTERVAL_NANOS = 20_000_000L; + private final RedisClient redisClient; + + public DefaultCreateOrderResultCache(RedisClient redisClient) { + this.redisClient = redisClient; + } + + @Override public BookOrderResponse get(String requestId) { + try { + return redisClient.get(key(requestId)); + } catch (RedisClientException redisFailure) { + LOGGER.warn("Create-order result cache unavailable, requestId={}", requestId, + redisFailure); + return null; + } + } + + @Override public void put(String requestId, BookOrderResponse response, long ttlMillis) { + try { + redisClient.set(key(requestId), response, ttlMillis); + } catch (RedisClientException redisFailure) { + // 缓存写失败不回滚已经提交的订单,后续请求仍可在锁内查询数据库回填。 + LOGGER.warn("Failed to cache create-order result, requestId={}", requestId, + redisFailure); + } + } + + @Override public BookOrderResponse await(String requestId, long waitMillis) { + long deadlineNanos = System.nanoTime() + waitMillis * 1_000_000L; + do { + BookOrderResponse response = get(requestId); + if (response != null) return response; + if (Thread.currentThread().isInterrupted()) return null; + LockSupport.parkNanos(POLL_INTERVAL_NANOS); + } while (System.nanoTime() < deadlineNanos); + return get(requestId); + } + + private static String key(String requestId) { return KEY_PREFIX + requestId; } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java new file mode 100644 index 0000000..d965d60 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java @@ -0,0 +1,18 @@ +package com.arch.policy.book.infrastructure.demo; + +import com.arch.policy.book.application.GdsBookingGateway; +import com.arch.policy.book.application.GdsBookingResult; + +public final class DemoGdsBookingGateway implements GdsBookingGateway { + @Override public GdsBookingResult createPnr(String orderNo) { + return GdsBookingResult.success("PNR-" + orderNo.substring(Math.max(0, orderNo.length() - 6))); + } + + @Override public GdsBookingResult queryPnr(String orderNo) { + return createPnr(orderNo); + } + + @Override public void cancelPnr(String orderNo, String childOrderNo, String pnr) { + // Demo adapter: production uses a GDS RPC client with the same idempotency key. + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java new file mode 100644 index 0000000..2094263 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java @@ -0,0 +1,15 @@ +package com.arch.policy.book.infrastructure.job; + +import com.arch.policy.book.application.OrderCreationOutboxPublisher; +import org.springframework.scheduling.annotation.Scheduled; + +public final class OrderCreationOutboxScheduler { + private final OrderCreationOutboxPublisher publisher; + + public OrderCreationOutboxScheduler(OrderCreationOutboxPublisher publisher) { + this.publisher = publisher; + } + + @Scheduled(fixedDelayString = "${book.creation.outbox-delay-ms:3000}") + public void publish() { publisher.publishPending(100); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java new file mode 100644 index 0000000..c0c01ef --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java @@ -0,0 +1,13 @@ +package com.arch.policy.book.infrastructure.job; + +import com.arch.policy.book.application.RecoveryTaskProcessor; +import org.springframework.scheduling.annotation.Scheduled; + +public final class OrderRecoveryScheduler { + private final RecoveryTaskProcessor processor; + + public OrderRecoveryScheduler(RecoveryTaskProcessor processor) { this.processor = processor; } + + @Scheduled(fixedDelayString = "${book.recovery.scan-delay-ms:5000}") + public void scan() { processor.processBatch(System.currentTimeMillis(), 100); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java index a76b4af..5895766 100644 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java @@ -1,7 +1,9 @@ package com.arch.policy.book.infrastructure.repository; import com.arch.policy.book.application.BookOrderStore; +import com.arch.policy.book.application.LocalCreateResult; import com.arch.policy.book.application.OrderOutboxMessage; +import com.arch.policy.book.application.OrderCreationOutboxMessage; import com.arch.policy.book.application.OrderStateHistory; import com.arch.policy.book.application.TransitionCommitResult; import com.arch.policy.book.domain.BookOrder; @@ -16,14 +18,18 @@ public final class InMemoryBookOrderStore implements BookOrderStore { private final Map orders = new HashMap(); private final Map requestIndexes = new HashMap(); private final Map eventResults = new HashMap(); + private final Map creationOutbox = + new HashMap(); private final List histories = new ArrayList(); private final List outboxMessages = new ArrayList(); + private int findByRequestIdCalls; @Override public synchronized BookOrder findByOrderNo(String orderNo) { return copy(orders.get(orderNo)); } @Override public synchronized BookOrder findByRequestId(String requestId) { + findByRequestIdCalls++; String orderNo = requestIndexes.get(requestId); return orderNo == null ? null : copy(orders.get(orderNo)); } @@ -32,15 +38,31 @@ public final class InMemoryBookOrderStore implements BookOrderStore { return copy(eventResults.get(eventId)); } - @Override public synchronized TransitionCommitResult create( - BookOrder order, OrderStateHistory history, OrderOutboxMessage outboxMessage) { + @Override public synchronized LocalCreateResult createOrder( + BookOrder order, OrderCreationOutboxMessage outboxMessage) { String existingOrderNo = requestIndexes.get(order.getRequestId()); if (existingOrderNo != null) { - return TransitionCommitResult.duplicate(copy(orders.get(existingOrderNo))); + return LocalCreateResult.duplicate(copy(orders.get(existingOrderNo))); } - persist(order, history, outboxMessage); + orders.put(order.getOrderNo(), order.copy()); requestIndexes.put(order.getRequestId(), order.getOrderNo()); - return TransitionCommitResult.committed(order.copy()); + creationOutbox.put(outboxMessage.getMessageId(), outboxMessage); + return LocalCreateResult.created(order.copy()); + } + + @Override public synchronized List findUnpublishedCreationOutbox( + int limit) { + List result = new ArrayList(); + for (OrderCreationOutboxMessage message : creationOutbox.values()) { + if (result.size() >= limit) break; + if (!message.isPublished()) result.add(message); + } + return result; + } + + @Override public synchronized void markCreationOutboxPublished(String messageId) { + OrderCreationOutboxMessage message = creationOutbox.get(messageId); + if (message != null) message.markPublished(); } @Override public synchronized TransitionCommitResult transit( @@ -66,6 +88,8 @@ public synchronized List historiesOf(String orderNo) { return Collections.unmodifiableList(result); } + public synchronized int getFindByRequestIdCalls() { return findByRequestIdCalls; } + public synchronized List unpublishedOutboxMessages() { List result = new ArrayList(); for (OrderOutboxMessage message : outboxMessages) { @@ -83,4 +107,5 @@ private void persist(BookOrder order, OrderStateHistory history, OrderOutboxMess } private static BookOrder copy(BookOrder order) { return order == null ? null : order.copy(); } + } diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java new file mode 100644 index 0000000..25124df --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java @@ -0,0 +1,17 @@ +package com.arch.policy.book.infrastructure.repository; + +import com.arch.policy.book.application.OrderWorkflowTask; +import com.arch.policy.book.application.OrderWorkflowTaskStore; + +import java.util.LinkedHashMap; +import java.util.Map; + +public final class InMemoryOrderWorkflowTaskStore implements OrderWorkflowTaskStore { + private final Map tasks = new LinkedHashMap(); + + @Override public synchronized void saveIfAbsent(OrderWorkflowTask task) { + if (!tasks.containsKey(task.getUniqueKey())) tasks.put(task.getUniqueKey(), task); + } + + public synchronized int size() { return tasks.size(); } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java new file mode 100644 index 0000000..c50d22b --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java @@ -0,0 +1,45 @@ +package com.arch.policy.book.infrastructure.repository; + +import com.arch.policy.book.application.RecoveryTask; +import com.arch.policy.book.application.RecoveryTaskStore; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public final class InMemoryRecoveryTaskStore implements RecoveryTaskStore { + private final Map tasks = new LinkedHashMap(); + + @Override public synchronized RecoveryTask saveIfAbsent(RecoveryTask task) { + RecoveryTask existing = tasks.get(task.getUniqueKey()); + if (existing != null) return existing.copy(); + tasks.put(task.getUniqueKey(), task.copy()); + return task.copy(); + } + + @Override public synchronized List findExecutable(long nowMillis, int limit) { + List result = new ArrayList(); + for (RecoveryTask task : tasks.values()) { + if (result.size() >= limit) break; + if ((task.getStatus() == RecoveryTask.RecoveryStatus.PENDING + || task.getStatus() == RecoveryTask.RecoveryStatus.RETRYING) + && task.getNextAttemptAtMillis() <= nowMillis) result.add(task.copy()); + } + return result; + } + + @Override public synchronized void save(RecoveryTask task) { + if (!tasks.containsKey(task.getUniqueKey())) { + throw new IllegalArgumentException("recovery task not found: " + task.getUniqueKey()); + } + tasks.put(task.getUniqueKey(), task.copy()); + } + + public synchronized int size() { return tasks.size(); } + + public synchronized RecoveryTask find(String uniqueKey) { + RecoveryTask task = tasks.get(uniqueKey); + return task == null ? null : task.copy(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java index 12625eb..869f952 100644 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java @@ -2,6 +2,7 @@ import com.arch.policy.api.book.BookOrderRpcService; import com.arch.policy.book.application.BookOrderApplicationService; +import com.arch.policy.book.application.OrderEventApplicationService; import com.arch.policy.common.book.BookOrderRequest; import com.arch.policy.common.book.BookOrderResponse; import com.arch.policy.common.book.OrderEventRequest; @@ -10,9 +11,12 @@ @DubboService(version = "1.0.0", timeout = 3000) public final class DubboBookOrderService implements BookOrderRpcService { private final BookOrderApplicationService applicationService; + private final OrderEventApplicationService eventService; - public DubboBookOrderService(BookOrderApplicationService applicationService) { + public DubboBookOrderService(BookOrderApplicationService applicationService, + OrderEventApplicationService eventService) { this.applicationService = applicationService; + this.eventService = eventService; } @Override public BookOrderResponse createOrder(BookOrderRequest request) { @@ -20,6 +24,6 @@ public DubboBookOrderService(BookOrderApplicationService applicationService) { } @Override public BookOrderResponse fireEvent(OrderEventRequest request) { - return applicationService.fireEvent(request); + return eventService.fireEvent(request); } } diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java new file mode 100644 index 0000000..1c22b60 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java @@ -0,0 +1,37 @@ +package com.arch.policy.book.infrastructure.seata; + +import com.arch.policy.book.application.OrderCreationSaga; +import io.seata.saga.engine.StateMachineEngine; +import io.seata.saga.statelang.domain.StateMachineInstance; + +import java.util.HashMap; +import java.util.Map; + +public final class SeataOrderCreationSaga implements OrderCreationSaga { + public static final String STATE_MACHINE_NAME = "BookOrderCreationSaga"; + private final StateMachineEngine stateMachineEngine; + private final String tenantId; + + public SeataOrderCreationSaga(StateMachineEngine stateMachineEngine, String tenantId) { + this.stateMachineEngine = stateMachineEngine; + this.tenantId = tenantId; + } + + /** + * 启动订单创建 Saga:调用 GDS 占编,并根据结果推进订单状态或登记补偿任务。 + */ + @Override public SagaStartResult start(String orderNo, String promotionId) { + // Saga 状态节点通过上下文读取订单号和促销活动 ID,作为服务方法的入参。 + Map context = new HashMap(); + context.put("orderNo", orderNo); + // Seata 表达式和服务参数统一使用空字符串,避免 null 在状态机流转中产生歧义。 + context.put("promotionId", promotionId == null ? "" : promotionId); + + // orderNo 作为业务键标识该订单的 Saga 实例,便于恢复任务关联同一笔订单流程。 + StateMachineInstance instance = stateMachineEngine.startWithBusinessKey( + STATE_MACHINE_NAME, tenantId, orderNo, context); + + // 向上层返回 Saga 实例 ID,以及状态机是否仍处于运行中。 + return new SagaStartResult(instance.getId(), instance.isRunning()); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java b/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java index d890783..08b6962 100644 --- a/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java +++ b/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java @@ -10,6 +10,7 @@ public final class BookOrderRequest implements Serializable { private String productId; private int quantity; private BigDecimal amount; + private String promotionId; public String getRequestId() { return requestId; } public void setRequestId(String requestId) { this.requestId = requestId; } @@ -21,4 +22,6 @@ public final class BookOrderRequest implements Serializable { public void setQuantity(int quantity) { this.quantity = quantity; } public BigDecimal getAmount() { return amount; } public void setAmount(BigDecimal amount) { this.amount = amount; } + public String getPromotionId() { return promotionId; } + public void setPromotionId(String promotionId) { this.promotionId = promotionId; } } diff --git a/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java index 71716c5..3dfe543 100644 --- a/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java +++ b/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java @@ -2,35 +2,132 @@ import com.arch.policy.book.application.BookOrderApplicationService; import com.arch.policy.book.application.BookOrderStore; +import com.arch.policy.book.application.CompensationTaskService; +import com.arch.policy.book.application.CreateOrderResultCache; +import com.arch.policy.book.application.DirectOrderCreationSaga; +import com.arch.policy.book.application.GdsBookingGateway; +import com.arch.policy.book.application.GdsReconciliationService; +import com.arch.policy.book.application.OrderCreationSaga; +import com.arch.policy.book.application.OrderCreationOutboxPublisher; +import com.arch.policy.book.application.OrderCreationSagaStateServices; +import com.arch.policy.book.application.OrderEventApplicationService; +import com.arch.policy.book.application.OrderWorkflowTaskStore; +import com.arch.policy.book.application.PaymentGateway; import com.arch.policy.book.application.PostTransitionExecutor; +import com.arch.policy.book.application.PromotionStockGateway; +import com.arch.policy.book.application.RecoveryTaskProcessor; +import com.arch.policy.book.application.RecoveryTaskStore; import com.arch.policy.book.application.RetryablePostTransitionExecutor; import com.arch.policy.book.domain.DefaultOrderTransitions; import com.arch.policy.book.domain.OrderStateMachine; +import com.arch.policy.book.infrastructure.demo.DemoGdsBookingGateway; +import com.arch.policy.book.infrastructure.job.OrderRecoveryScheduler; +import com.arch.policy.book.infrastructure.job.OrderCreationOutboxScheduler; import com.arch.policy.book.infrastructure.repository.InMemoryBookOrderStore; import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; +import com.arch.policy.book.infrastructure.repository.InMemoryOrderWorkflowTaskStore; +import com.arch.policy.book.infrastructure.repository.InMemoryRecoveryTaskStore; +import com.arch.policy.book.infrastructure.cache.DefaultCreateOrderResultCache; +import com.arch.policy.common.infrastructure.redis.RedissonDistributedLock; +import com.arch.policy.common.infrastructure.redis.RedissonRedisClient; +import com.arch.policy.common.lock.DistributedLock; +import com.arch.policy.common.redis.RedisClient; +import com.arch.policy.book.infrastructure.seata.SeataOrderCreationSaga; +import io.seata.saga.engine.StateMachineEngine; +import org.redisson.api.RedissonClient; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; @Configuration +@EnableScheduling public class BookOrderConfiguration { @Bean public OrderStateMachine orderStateMachine() { return new OrderStateMachine(DefaultOrderTransitions.definitions()); } @Bean public BookOrderStore bookOrderStore() { return new InMemoryBookOrderStore(); } - + @Bean public DistributedLock distributedLock(RedissonClient redisson) { + return new RedissonDistributedLock(redisson); + } + @Bean public RedisClient redisClient(RedissonClient redisson) { + return new RedissonRedisClient(redisson); + } + @Bean public CreateOrderResultCache createOrderResultCache(RedisClient redisClient) { + return new DefaultCreateOrderResultCache(redisClient); + } + @Bean public RecoveryTaskStore recoveryTaskStore() { return new InMemoryRecoveryTaskStore(); } + @Bean public OrderWorkflowTaskStore orderWorkflowTaskStore() { + return new InMemoryOrderWorkflowTaskStore(); + } @Bean public InMemoryFailedPostActionStore failedPostActionStore() { return new InMemoryFailedPostActionStore(); } - @Bean public PostTransitionExecutor postTransitionExecutor( InMemoryFailedPostActionStore failedPostActionStore) { return new RetryablePostTransitionExecutor(failedPostActionStore); } - - @Bean public BookOrderApplicationService bookOrderApplicationService( + @Bean public OrderEventApplicationService orderEventApplicationService( BookOrderStore store, OrderStateMachine stateMachine, - PostTransitionExecutor postTransitionExecutor) { - return new BookOrderApplicationService(store, stateMachine, postTransitionExecutor); + PostTransitionExecutor postTransitionExecutor, + CompensationTaskService compensationTaskService, + CreateOrderResultCache resultCache) { + return new OrderEventApplicationService(store, stateMachine, postTransitionExecutor, + compensationTaskService, resultCache); + } + @Bean public GdsBookingGateway gdsBookingGateway() { return new DemoGdsBookingGateway(); } + @Bean public PromotionStockGateway promotionStockGateway() { + return (orderNo, promotionId) -> { }; + } + @Bean public PaymentGateway paymentGateway() { return (orderNo, paymentNo) -> { }; } + @Bean public CompensationTaskService compensationTaskService(RecoveryTaskStore store) { + return new CompensationTaskService(store); + } + @Bean(name = "orderCreationSagaStateServices") + public OrderCreationSagaStateServices orderCreationSagaStateServices( + BookOrderStore store, GdsBookingGateway gdsGateway, + OrderEventApplicationService eventService, CompensationTaskService compensationService, + OrderWorkflowTaskStore workflowTaskStore) { + return new OrderCreationSagaStateServices(store, gdsGateway, eventService, + compensationService, workflowTaskStore); + } + @Bean public OrderCreationSaga orderCreationSaga( + ObjectProvider engineProvider, + OrderCreationSagaStateServices services, + @Value("${book.seata.tenant-id:book}") String tenantId) { + StateMachineEngine engine = engineProvider.getIfAvailable(); + return engine == null ? new DirectOrderCreationSaga(services) + : new SeataOrderCreationSaga(engine, tenantId); + } + @Bean public BookOrderApplicationService bookOrderApplicationService( + BookOrderStore store, OrderCreationOutboxPublisher outboxPublisher, + DistributedLock distributedLock, + CreateOrderResultCache resultCache) { + return new BookOrderApplicationService(store, outboxPublisher, distributedLock, + resultCache); + } + @Bean public OrderCreationOutboxPublisher orderCreationOutboxPublisher( + BookOrderStore store, OrderCreationSaga saga) { + return new OrderCreationOutboxPublisher(store, saga); + } + @Bean public OrderCreationOutboxScheduler orderCreationOutboxScheduler( + OrderCreationOutboxPublisher publisher) { + return new OrderCreationOutboxScheduler(publisher); + } + @Bean public GdsReconciliationService gdsReconciliationService( + GdsBookingGateway gdsGateway, OrderCreationSagaStateServices sagaStates) { + return new GdsReconciliationService(gdsGateway, sagaStates); + } + @Bean public RecoveryTaskProcessor recoveryTaskProcessor( + RecoveryTaskStore taskStore, PromotionStockGateway promotionGateway, + GdsBookingGateway gdsGateway, PaymentGateway paymentGateway, + GdsReconciliationService reconciliationService) { + return new RecoveryTaskProcessor(taskStore, promotionGateway, gdsGateway, + paymentGateway, reconciliationService); + } + @Bean public OrderRecoveryScheduler orderRecoveryScheduler(RecoveryTaskProcessor processor) { + return new OrderRecoveryScheduler(processor); } } diff --git a/architecture/src/main/java/com/arch/policy/common/config/RedissonConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/RedissonConfiguration.java new file mode 100644 index 0000000..5fb5cc6 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/config/RedissonConfiguration.java @@ -0,0 +1,28 @@ +package com.arch.policy.common.config; + +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.config.Config; +import org.redisson.config.SingleServerConfig; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class RedissonConfiguration { + @Bean(destroyMethod = "shutdown") + @ConditionalOnMissingBean(RedissonClient.class) + public RedissonClient redissonClient( + @Value("${spring.redis.host:localhost}") String host, + @Value("${spring.redis.port:6379}") int port, + @Value("${spring.redis.database:0}") int database, + @Value("${spring.redis.password:}") String password) { + Config config = new Config(); + SingleServerConfig server = config.useSingleServer() + .setAddress("redis://" + host + ":" + port) + .setDatabase(database); + if (password != null && !password.trim().isEmpty()) server.setPassword(password); + return Redisson.create(config); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java new file mode 100644 index 0000000..2aa68e8 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java @@ -0,0 +1,54 @@ +package com.arch.policy.common.config; + +import io.seata.saga.engine.StateMachineEngine; +import io.seata.saga.engine.config.DbStateMachineConfig; +import io.seata.saga.engine.impl.ProcessCtrlStateMachineEngine; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.sql.DataSource; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Configuration +public class SeataSagaConfiguration { + @Bean(destroyMethod = "shutdown") + @ConditionalOnBean(DataSource.class) + public ThreadPoolExecutor seataSagaExecutor() { + return new ThreadPoolExecutor(2, 16, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue(1000), new ThreadPoolExecutor.CallerRunsPolicy()); + } + + @Bean + @ConditionalOnBean(DataSource.class) + public DbStateMachineConfig dbStateMachineConfig( + DataSource dataSource, ThreadPoolExecutor seataSagaExecutor, + @Value("${spring.application.name:architecture}") String applicationId, + @Value("${book.seata.tx-service-group:book-saga-group}") String txServiceGroup, + @Value("${book.seata.tenant-id:book}") String tenantId) { + DbStateMachineConfig config = new DbStateMachineConfig(); + config.setDataSource(dataSource); + config.setApplicationId(applicationId); + config.setTxServiceGroup(txServiceGroup); + config.setDefaultTenantId(tenantId); + config.setThreadPoolExecutor(seataSagaExecutor); + config.setAutoRegisterResources(true); + config.setResources(new String[] { "classpath*:statelang/book_order_creation_saga.json" }); + config.setSagaJsonParser("jackson"); + config.setSagaBranchRegisterEnable(true); + return config; + } + + @Bean + @ConditionalOnBean(DbStateMachineConfig.class) + @ConditionalOnMissingBean(StateMachineEngine.class) + public StateMachineEngine stateMachineEngine(DbStateMachineConfig config) { + ProcessCtrlStateMachineEngine engine = new ProcessCtrlStateMachineEngine(); + engine.setStateMachineConfig(config); + return engine; + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonDistributedLock.java b/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonDistributedLock.java new file mode 100644 index 0000000..810190c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonDistributedLock.java @@ -0,0 +1,47 @@ +package com.arch.policy.common.infrastructure.redis; + +import com.arch.policy.common.lock.DistributedLock; +import com.arch.policy.common.lock.DistributedLockUnavailableException; +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.redisson.client.RedisException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +public final class RedissonDistributedLock implements DistributedLock { + private static final Logger LOGGER = LoggerFactory.getLogger(RedissonDistributedLock.class); + private final RedissonClient redisson; + + public RedissonDistributedLock(RedissonClient redisson) { this.redisson = redisson; } + + @Override public LockHandle tryAcquire(String key, long waitMillis) { + RLock lock = redisson.getLock(key); + try { + // 不指定 leaseTime,交由 Redisson watchdog 在持锁线程存活期间自动续期。 + boolean acquired = lock.tryLock(waitMillis, TimeUnit.MILLISECONDS); + return acquired ? new RedissonLockHandle(lock) : null; + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return null; + } catch (RedisException redisFailure) { + throw new DistributedLockUnavailableException( + "Redisson lock unavailable, key=" + key, redisFailure); + } + } + + private static final class RedissonLockHandle implements LockHandle { + private final RLock lock; + + private RedissonLockHandle(RLock lock) { this.lock = lock; } + + @Override public void close() { + try { + if (lock.isHeldByCurrentThread()) lock.unlock(); + } catch (RedisException redisFailure) { + LOGGER.warn("Failed to release Redisson lock, key={}", lock.getName(), redisFailure); + } + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonRedisClient.java b/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonRedisClient.java new file mode 100644 index 0000000..599ce29 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/infrastructure/redis/RedissonRedisClient.java @@ -0,0 +1,30 @@ +package com.arch.policy.common.infrastructure.redis; + +import com.arch.policy.common.redis.RedisClient; +import com.arch.policy.common.redis.RedisClientException; +import org.redisson.api.RedissonClient; +import org.redisson.client.RedisException; + +import java.util.concurrent.TimeUnit; + +public final class RedissonRedisClient implements RedisClient { + private final RedissonClient redisson; + + public RedissonRedisClient(RedissonClient redisson) { this.redisson = redisson; } + + @Override public T get(String key) { + try { + return redisson.getBucket(key).get(); + } catch (RedisException redisFailure) { + throw new RedisClientException("Failed to read Redis key=" + key, redisFailure); + } + } + + @Override public void set(String key, Object value, long ttlMillis) { + try { + redisson.getBucket(key).set(value, ttlMillis, TimeUnit.MILLISECONDS); + } catch (RedisException redisFailure) { + throw new RedisClientException("Failed to write Redis key=" + key, redisFailure); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/lock/DistributedLock.java b/architecture/src/main/java/com/arch/policy/common/lock/DistributedLock.java new file mode 100644 index 0000000..dbd5277 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/lock/DistributedLock.java @@ -0,0 +1,14 @@ +package com.arch.policy.common.lock; + +public interface DistributedLock { + /** + * 尝试获取指定 key 对应的分布式锁。 + * + * @return 锁句柄;等待超时返回 null + */ + LockHandle tryAcquire(String key, long waitMillis); + + interface LockHandle extends AutoCloseable { + @Override void close(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/lock/DistributedLockUnavailableException.java b/architecture/src/main/java/com/arch/policy/common/lock/DistributedLockUnavailableException.java new file mode 100644 index 0000000..1e8828f --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/lock/DistributedLockUnavailableException.java @@ -0,0 +1,7 @@ +package com.arch.policy.common.lock; + +public final class DistributedLockUnavailableException extends RuntimeException { + public DistributedLockUnavailableException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/architecture/src/main/java/com/arch/policy/common/redis/RedisClient.java b/architecture/src/main/java/com/arch/policy/common/redis/RedisClient.java new file mode 100644 index 0000000..a8a2f88 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/redis/RedisClient.java @@ -0,0 +1,7 @@ +package com.arch.policy.common.redis; + +public interface RedisClient { + T get(String key); + + void set(String key, Object value, long ttlMillis); +} diff --git a/architecture/src/main/java/com/arch/policy/common/redis/RedisClientException.java b/architecture/src/main/java/com/arch/policy/common/redis/RedisClientException.java new file mode 100644 index 0000000..15dfdcf --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/common/redis/RedisClientException.java @@ -0,0 +1,7 @@ +package com.arch.policy.common.redis; + +public final class RedisClientException extends RuntimeException { + public RedisClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/architecture/src/main/resources/application.yml b/architecture/src/main/resources/application.yml index c30aadb..02c14bb 100644 --- a/architecture/src/main/resources/application.yml +++ b/architecture/src/main/resources/application.yml @@ -20,6 +20,15 @@ dubbo: name: dubbo port: 20880 +book: + seata: + tenant-id: book + tx-service-group: book-saga-group + recovery: + scan-delay-ms: 5000 + creation: + scan-delay-ms: 3000 + policy: search: redis-ttl-seconds: 60 diff --git a/architecture/src/main/resources/db/book_order_schema.sql b/architecture/src/main/resources/db/book_order_schema.sql new file mode 100644 index 0000000..4de28ff --- /dev/null +++ b/architecture/src/main/resources/db/book_order_schema.sql @@ -0,0 +1,37 @@ +CREATE TABLE book_order_recovery_task ( + task_id BIGINT PRIMARY KEY AUTO_INCREMENT, + unique_key VARCHAR(256) NOT NULL, + order_no VARCHAR(64) NOT NULL, + child_order_no VARCHAR(64) NOT NULL DEFAULT '', + pnr VARCHAR(32) NOT NULL DEFAULT '', + task_type VARCHAR(32) NOT NULL, + task_status VARCHAR(32) NOT NULL, + attempts INT NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMP NULL, + last_error VARCHAR(1024), + UNIQUE KEY uk_recovery_business (unique_key), + UNIQUE KEY uk_cancel_pnr (order_no, child_order_no, pnr, task_type) +); + +CREATE TABLE book_order_state_history ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + event_id VARCHAR(128) NOT NULL, + order_no VARCHAR(64) NOT NULL, + from_state VARCHAR(32) NOT NULL, + event_type VARCHAR(64) NOT NULL, + to_state VARCHAR(32) NOT NULL, + operator VARCHAR(64) NOT NULL, + occurred_at TIMESTAMP NOT NULL, + UNIQUE KEY uk_order_event (event_id) +); + +CREATE TABLE book_order_outbox ( + id BIGINT PRIMARY KEY AUTO_INCREMENT, + event_id VARCHAR(128) NOT NULL, + order_no VARCHAR(64) NOT NULL, + event_type VARCHAR(64) NOT NULL, + payload TEXT NOT NULL, + published TINYINT NOT NULL DEFAULT 0, + created_at TIMESTAMP NOT NULL, + UNIQUE KEY uk_outbox_event (event_id) +); diff --git a/architecture/src/main/resources/statelang/book_order_creation_saga.json b/architecture/src/main/resources/statelang/book_order_creation_saga.json new file mode 100644 index 0000000..c6284bd --- /dev/null +++ b/architecture/src/main/resources/statelang/book_order_creation_saga.json @@ -0,0 +1,54 @@ +{ + "Name": "BookOrderCreationSaga", + "Comment": "本地订单CREATE提交后执行GDS占编,并按SUCCESS/FAIL/UNKNOWN分流", + "Version": "1.0.0", + "StartState": "ReservePnr", + "States": { + "ReservePnr": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "reservePnr", + "Input": ["$.[orderNo]"], + "Output": {"gdsResult": "$#root"}, + "CompensateState": "CancelReservedPnr", + "Catch": [{"Exceptions": ["java.lang.Throwable"], "Next": "MarkUnknown"}], + "Next": "RouteGdsResult" + }, + "RouteGdsResult": { + "Type": "Choice", + "Choices": [ + {"Expression": "$.[gdsResult].status.name() == 'SUCCESS'", "Next": "MarkSuccess"}, + {"Expression": "$.[gdsResult].status.name() == 'FAIL'", "Next": "MarkFailure"} + ], + "Default": "MarkUnknown" + }, + "MarkSuccess": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "markCreateSuccess", + "Input": ["$.[orderNo]", "$.[gdsResult].pnr"], + "Next": "Succeed" + }, + "MarkFailure": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "markCreateFailure", + "Input": ["$.[orderNo]", "$.[gdsResult].errorCode", "$.[promotionId]"], + "Next": "Succeed" + }, + "MarkUnknown": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "markCreateUnknown", + "Input": ["$.[orderNo]", "GDS_RESULT_UNKNOWN"], + "Next": "Succeed" + }, + "CancelReservedPnr": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "enqueueCancelPnr", + "Input": ["$.[orderNo]", "", "$.[gdsResult].pnr"] + }, + "Succeed": {"Type": "Succeed"} + } +} diff --git a/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java b/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java deleted file mode 100644 index 6dffeaf..0000000 --- a/architecture/src/test/java/com/arch/policy/book/application/BookOrderApplicationServiceTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.DefaultOrderTransitions; -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.infrastructure.repository.InMemoryBookOrderStore; -import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; -import com.arch.policy.common.book.BookOrderRequest; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.book.OrderEventRequest; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.math.BigDecimal; -import java.util.Collections; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class BookOrderApplicationServiceTest { - private InMemoryBookOrderStore store; - private BookOrderApplicationService service; - - @BeforeEach - void setUp() { - store = new InMemoryBookOrderStore(); - OrderStateMachine stateMachine = new OrderStateMachine(DefaultOrderTransitions.definitions()); - service = new BookOrderApplicationService(store, stateMachine, - new RetryablePostTransitionExecutor(new InMemoryFailedPostActionStore())); - } - - @Test - void createsOrderIdempotentlyAndCommitsHistoryWithOutbox() { - BookOrderRequest request = request("request-1"); - - BookOrderResponse first = service.createOrder(request); - BookOrderResponse repeated = service.createOrder(request); - - assertEquals("WAIT_PAY", first.getState()); - assertEquals(1L, first.getVersion()); - assertEquals(first.getOrderNo(), repeated.getOrderNo()); - assertEquals(1, store.historiesOf(first.getOrderNo()).size()); - assertEquals(1, store.unpublishedOutboxMessages().size()); - } - - @Test - void drivesOrderByBusinessEventsInsteadOfTargetStates() { - BookOrderResponse created = service.createOrder(request("request-2")); - - BookOrderResponse booking = fire(created, "pay-1", "PAY_SUCCEEDED"); - BookOrderResponse booked = fire(booking, "book-1", "BOOK_SUCCEEDED"); - - assertEquals("BOOKED", booked.getState()); - assertEquals(3L, booked.getVersion()); - assertEquals(3, store.historiesOf(booked.getOrderNo()).size()); - assertEquals(3, store.unpublishedOutboxMessages().size()); - } - - @Test - void returnsOriginalResultForDuplicateEventEvenWithStaleVersion() { - BookOrderResponse created = service.createOrder(request("request-3")); - BookOrderResponse booking = fire(created, "pay-2", "PAY_SUCCEEDED"); - - BookOrderResponse duplicate = fire(created, "pay-2", "PAY_SUCCEEDED"); - - assertEquals(booking.getState(), duplicate.getState()); - assertEquals(booking.getVersion(), duplicate.getVersion()); - assertEquals(2, store.historiesOf(created.getOrderNo()).size()); - } - - @Test - void rejectsUnsupportedEventAndConcurrentUpdate() { - BookOrderResponse created = service.createOrder(request("request-4")); - assertThrows(IllegalStateException.class, - () -> fire(created, "book-too-early", "BOOK_SUCCEEDED")); - - fire(created, "pay-3", "PAY_SUCCEEDED"); - assertThrows(IllegalStateException.class, - () -> fire(created, "cancel-stale", "CANCEL")); - } - - private BookOrderResponse fire(BookOrderResponse current, String eventId, String event) { - OrderEventRequest request = new OrderEventRequest(); - request.setEventId(eventId); - request.setOrderNo(current.getOrderNo()); - request.setExpectedVersion(current.getVersion()); - request.setEvent(event); - request.setOperator("test-operator"); - if ("PAY_SUCCEEDED".equals(event)) { - request.setAttributes(Collections.singletonMap("paymentNo", "payment-1")); - } else if ("BOOK_SUCCEEDED".equals(event)) { - request.setAttributes(Collections.singletonMap("bookingReference", "PNR001")); - } - return service.fireEvent(request); - } - - private static BookOrderRequest request(String requestId) { - BookOrderRequest request = new BookOrderRequest(); - request.setRequestId(requestId); - request.setCustomerId("customer-1"); - request.setProductId("policy-1"); - request.setQuantity(1); - request.setAmount(new BigDecimal("100.00")); - return request; - } -} diff --git a/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java b/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java deleted file mode 100644 index 892f71f..0000000 --- a/architecture/src/test/java/com/arch/policy/book/application/RetryablePostTransitionExecutorTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; -import com.arch.policy.book.domain.OrderEvent; -import com.arch.policy.book.domain.OrderState; -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.domain.OrderTransition; -import com.arch.policy.book.domain.StateTransitionContext; -import com.arch.policy.book.domain.TransitionExecution; -import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; -import org.junit.jupiter.api.Test; - -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Collections; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class RetryablePostTransitionExecutorTest { - @Test - void recordsPostActionFailureForRetryWithoutRevertingCommittedState() { - OrderTransition transition = OrderTransition - .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) - .after(context -> { throw new IllegalStateException("notification unavailable"); }) - .build(); - OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); - BookOrder order = BookOrder.create("order-1", "request-1", "customer-1", "product-1", - 1, BigDecimal.ONE); - TransitionExecution execution = stateMachine.fire(new StateTransitionContext( - "event-1", OrderEvent.CREATE_SUCCEEDED, order, "tester", - Collections.emptyMap())); - InMemoryFailedPostActionStore failureStore = new InMemoryFailedPostActionStore(); - - new RetryablePostTransitionExecutor(failureStore).execute(stateMachine, execution); - - assertEquals(OrderState.WAIT_PAY, order.getState()); - assertEquals(1, failureStore.all().size()); - assertEquals("event-1", failureStore.all().get(0).getEventId()); - } -} diff --git a/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java b/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java deleted file mode 100644 index 9b17b6f..0000000 --- a/architecture/src/test/java/com/arch/policy/book/domain/OrderStateMachineTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.arch.policy.book.domain; - -import org.junit.jupiter.api.Test; - -import java.math.BigDecimal; -import java.util.Arrays; -import java.util.Collections; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class OrderStateMachineTest { - @Test - void resolvesTargetByCurrentStateAndBusinessEvent() { - OrderStateMachine stateMachine = new OrderStateMachine(DefaultOrderTransitions.definitions()); - - assertEquals(OrderState.WAIT_PAY, - stateMachine.targetOf(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED)); - assertEquals(OrderState.BOOKING, - stateMachine.targetOf(OrderState.WAIT_PAY, OrderEvent.PAY_SUCCEEDED)); - assertEquals(OrderState.BOOKED, - stateMachine.targetOf(OrderState.VALIDATE_FAIL, OrderEvent.VALIDATE_SUCCEEDED)); - assertNull(stateMachine.targetOf(OrderState.WAIT_PAY, OrderEvent.BOOK_SUCCEEDED)); - } - - @Test - void executesGuardAndPreActionBeforeMutationAndPostActionAfterCommit() { - final AtomicInteger calls = new AtomicInteger(); - TransitionGuard guard = context -> calls.compareAndSet(0, 1); - TransitionAction preAction = context -> { - assertEquals(OrderState.CREATE, context.getOrder().getState()); - calls.compareAndSet(1, 2); - }; - TransitionAction postAction = context -> { - assertEquals(OrderState.WAIT_PAY, context.getOrder().getState()); - calls.compareAndSet(2, 3); - }; - OrderTransition transition = OrderTransition - .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) - .guard(guard).before(preAction).after(postAction).build(); - OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); - BookOrder order = order(); - - TransitionExecution execution = stateMachine.fire(context(order, OrderEvent.CREATE_SUCCEEDED)); - assertEquals(2, calls.get()); - assertEquals(OrderState.WAIT_PAY, order.getState()); - - stateMachine.afterCommit(execution); - assertEquals(3, calls.get()); - } - - @Test - void guardFailurePreventsStateMutation() { - OrderTransition transition = OrderTransition - .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY) - .guard(context -> { throw new IllegalStateException("risk rejected"); }) - .build(); - OrderStateMachine stateMachine = new OrderStateMachine(Arrays.asList(transition)); - BookOrder order = order(); - - assertThrows(IllegalStateException.class, - () -> stateMachine.fire(context(order, OrderEvent.CREATE_SUCCEEDED))); - assertEquals(OrderState.CREATE, order.getState()); - assertEquals(0L, order.getVersion()); - } - - @Test - void rejectsDuplicateTransitionRegistration() { - OrderTransition first = OrderTransition - .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY).build(); - OrderTransition duplicate = OrderTransition - .from(OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.CREATE_FAIL).build(); - - assertThrows(IllegalStateException.class, - () -> new OrderStateMachine(Arrays.asList(first, duplicate))); - } - - private static BookOrder order() { - return BookOrder.create("order-1", "request-1", "customer-1", "product-1", - 1, BigDecimal.ONE); - } - - private static StateTransitionContext context(BookOrder order, OrderEvent event) { - return new StateTransitionContext("event-1", event, order, "tester", - Collections.emptyMap()); - } -} diff --git a/architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java b/architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java deleted file mode 100644 index 8117b2a..0000000 --- a/architecture/src/test/java/com/arch/policy/search/application/AsyncSearchCoordinatorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.arch.policy.search.application; - -import com.arch.policy.common.search.PolicySearchRequest; -import com.arch.policy.common.search.PolicySearchResponse; -import com.arch.policy.common.search.SupplierCallbackRequest; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; - -import java.util.*; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -class AsyncSearchCoordinatorTest { - @Test - void waitsForAllSuppliersAndAggregatesAsyncCallbacks() throws Exception { - final MemoryStore store = new MemoryStore(); - final LocalSearchWaiters waiters = new LocalSearchWaiters(); - final ObjectMapper mapper = new ObjectMapper(); - SupplierTaskDispatcher dispatcher = (searchKey, suppliers, request) -> { - new Thread(() -> { - try { - for (String supplier : suppliers) { - SupplierCallbackRequest callback = callback(searchKey, supplier, supplier + "-quote", true); - store.recordCallback(searchKey, supplier, mapper.writeValueAsString(callback), true, 60); - } - waiters.signal(searchKey); - } catch (Exception failure) { throw new RuntimeException(failure); } - }).start(); - }; - AsyncSearchCoordinator coordinator = new AsyncSearchCoordinator(store, dispatcher, waiters, mapper, 60); - PolicySearchRequest request = request(1000, "A", "B"); - - PolicySearchResponse response = coordinator.search(request); - - assertEquals(SearchState.COMPLETED.name(), response.getState()); - assertEquals(2, response.getResults().size()); - } - - @Test - void timeoutReturnsPartialResultsAndRejectsLateCallbacks() throws Exception { - final MemoryStore store = new MemoryStore(); - final ObjectMapper mapper = new ObjectMapper(); - SupplierTaskDispatcher dispatcher = (searchKey, suppliers, request) -> { - try { - SupplierCallbackRequest partial = callback(searchKey, "A", "A-quote", true); - store.recordCallback(searchKey, "A", mapper.writeValueAsString(partial), true, 60); - } catch (Exception failure) { throw new RuntimeException(failure); } - }; - AsyncSearchCoordinator coordinator = new AsyncSearchCoordinator( - store, dispatcher, new LocalSearchWaiters(), mapper, 60); - - PolicySearchResponse response = coordinator.search(request(30, "A", "B")); - - assertEquals(SearchState.TIMED_OUT.name(), response.getState()); - assertEquals(1, response.getResults().size()); - } - - private static PolicySearchRequest request(long timeout, String... suppliers) { - PolicySearchRequest request = new PolicySearchRequest(); - request.setCriteria("demo"); - request.setSupplierIds(Arrays.asList(suppliers)); - request.setTotalTimeoutMillis(timeout); - return request; - } - - private static SupplierCallbackRequest callback(String key, String supplier, String result, boolean done) { - SupplierCallbackRequest request = new SupplierCallbackRequest(); - request.setSearchKey(key); - request.setSupplierId(supplier); - request.setResults(Collections.singletonList(result)); - request.setSearchFinished(done); - return request; - } - - private static final class MemoryStore implements SearchStateStore { - private SearchState state; - private final Set pending = new HashSet(); - private final List results = new ArrayList(); - @Override public synchronized void initialize(String key, Set suppliers, long ttl) { - pending.addAll(suppliers); - state = suppliers.isEmpty() ? SearchState.COMPLETED : SearchState.WAITING; - } - @Override public synchronized SearchState getState(String key) { return state; } - @Override public synchronized List getResultPayloads(String key) { - return new ArrayList(results); - } - @Override public synchronized void recordCallback(String key, String supplier, String payload, - boolean finished, long ttl) { - if (state != SearchState.WAITING) return; - if (!payload.isEmpty()) results.add(payload); - if (finished) pending.remove(supplier); - if (pending.isEmpty()) state = SearchState.COMPLETED; - } - @Override public synchronized void markTimedOut(String key, long ttl) { - if (state == SearchState.WAITING) state = SearchState.TIMED_OUT; - } - } -} diff --git a/architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java b/architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java deleted file mode 100644 index 9a1ceaf..0000000 --- a/architecture/src/test/java/com/arch/policy/search/domain/snapshot/SnapshotLifecycleTest.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.arch.policy.search.domain.snapshot; - -import com.arch.policy.common.model.MessagePosition; -import com.arch.policy.common.model.PolicyRecord; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -import static com.arch.policy.search.domain.snapshot.SnapshotPorts.*; -import static org.junit.jupiter.api.Assertions.*; - -class SnapshotLifecycleTest { - @TempDir Path root; - - @Test - void buildsFullDataReplaysChangesAndValidatesPosition() throws Exception { - SnapshotBuilder builder = builder(false); - - PolicySnapshot snapshot = builder.build("V101"); - - assertEquals(new MessagePosition(12), snapshot.getPosition()); - assertArrayEquals(bytes("new-detail"), snapshot.findDetail(1)); - assertEquals(1, snapshot.findPolicyIds("active").getCardinality()); - assertEquals(0, snapshot.findPolicyIds("draft").getCardinality()); - snapshot.closeAndDelete(); - } - - @Test - void switchWaitsForOldQueriesBeforeClosingAndDeletingOldVersion() throws Exception { - ActiveSnapshotRegistry registry = new ActiveSnapshotRegistry(); - PolicySnapshot old = builder(false).build("V100"); - PolicySnapshot replacement = builder(false).build("V101"); - registry.activate(old); - ActiveSnapshotRegistry.SnapshotLease oldQuery = registry.acquire(); - - registry.activate(replacement); - - assertTrue(Files.exists(old.getDirectory())); - assertArrayEquals(bytes("new-detail"), oldQuery.snapshot().findDetail(1)); - try (ActiveSnapshotRegistry.SnapshotLease newQuery = registry.acquire()) { - assertEquals("V101", newQuery.snapshot().getVersion()); - } - oldQuery.close(); - assertFalse(Files.exists(old.getDirectory())); - assertThrows(IllegalStateException.class, () -> old.findPolicyIds("active")); - registry.close(); - } - - @Test - void failedRuntimeCandidateDoesNotReplaceServingSnapshot() throws Exception { - ActiveSnapshotRegistry registry = new ActiveSnapshotRegistry(); - registry.activate(builder(false).build("V100")); - ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); - PolicySnapshotService service = new PolicySnapshotService(builder(true), registry, executor, 1); - - assertTrue(service.refresh("V101")); - waitUntilBuildFinishes(service); - - assertTrue(service.isReady()); - try (ActiveSnapshotRegistry.SnapshotLease query = registry.acquire()) { - assertEquals("V100", query.snapshot().getVersion()); - } - assertFalse(Files.exists(root.resolve("V101"))); - service.close(); - } - - private SnapshotBuilder builder(final boolean failValidation) { - FullPolicyLoader full = target -> { - target.upsert(new PolicyRecord(1, bytes("old-detail"), Collections.singleton("draft"))); - return new MessagePosition(10); - }; - IncrementalReplayer replay = new IncrementalReplayer() { - @Override public MessagePosition latestPosition() { return new MessagePosition(12); } - @Override public MessagePosition replayInto(PolicySnapshot target, MessagePosition from, - MessagePosition latest) throws Exception { - target.upsert(new PolicyRecord(1, bytes("new-detail"), - Collections.singleton("active"))); - return latest; - } - }; - SnapshotValidator validator = (candidate, position) -> { - new DefaultSnapshotValidator().validate(candidate, position); - if (failValidation) throw new IllegalStateException("invalid candidate"); - }; - SnapshotDirectory directories = new SnapshotDirectory() { - @Override public Path create(String version) throws Exception { - Path path = root.resolve(version); - Files.createDirectories(path); - return path; - } - @Override public void delete(Path directory) throws Exception { - if (!Files.exists(directory)) return; - Files.walk(directory).sorted(Comparator.reverseOrder()).forEach(path -> { - try { Files.deleteIfExists(path); } - catch (Exception failure) { throw new RuntimeException(failure); } - }); - } - }; - return new SnapshotBuilder(full, replay, validator, directories); - } - - private static byte[] bytes(String value) { - return value.getBytes(java.nio.charset.StandardCharsets.UTF_8); - } - - private static void waitUntilBuildFinishes(PolicySnapshotService service) throws Exception { - long deadline = System.currentTimeMillis() + 3000; - while (System.currentTimeMillis() < deadline) { - if (!service.isBuilding()) return; - Thread.sleep(10); - } - fail("candidate build did not finish"); - } -} From 17f1b09d7c2f2e842ea16cc08eaf7aef9c529807 Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 22 Jul 2026 21:59:01 +0800 Subject: [PATCH 36/37] =?UTF-8?q?=E4=B8=8B=E5=8D=95=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=9A=E4=B8=8B=E5=8D=95+=E5=BA=93?= =?UTF-8?q?=E5=AD=98=E6=89=A3=E5=87=8F+=E4=B8=89=E6=96=B9=E4=B8=8B?= =?UTF-8?q?=E5=8D=95=EF=BC=8C=E4=BD=BF=E7=94=A8Seata=20saga=E7=AE=A1?= =?UTF-8?q?=E6=8E=A7=E4=BA=8B=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../policy/api/book/BookOrderRpcService.java | 12 -- .../policy/api/order/OrderRpcService.java | 14 ++ .../BookOrderApplicationService.java | 153 ------------------ .../book/application/BookOrderStore.java | 22 --- .../application/CompensationTaskService.java | 51 ------ .../application/CreateOrderResultCache.java | 11 -- .../application/DirectOrderCreationSaga.java | 24 --- .../application/ExternalResultStatus.java | 7 - .../book/application/GdsBookingGateway.java | 9 -- .../book/application/GdsBookingResult.java | 29 ---- .../application/GdsReconciliationService.java | 25 --- .../book/application/LocalCreateResult.java | 24 --- .../OrderCreationOutboxMessage.java | 21 --- .../OrderCreationOutboxPublisher.java | 25 --- .../book/application/OrderCreationSaga.java | 20 --- .../OrderCreationSagaStateServices.java | 86 ---------- .../OrderEventApplicationService.java | 97 ----------- .../book/application/OrderOutboxMessage.java | 30 ---- .../book/application/OrderReconciler.java | 5 - .../book/application/OrderStateHistory.java | 33 ---- .../book/application/OrderWorkflowTask.java | 19 --- .../application/OrderWorkflowTaskStore.java | 5 - .../book/application/PaymentGateway.java | 5 - .../application/PostTransitionExecutor.java | 8 - .../application/PromotionStockGateway.java | 5 - .../policy/book/application/RecoveryTask.java | 66 -------- .../application/RecoveryTaskProcessor.java | 67 -------- .../book/application/RecoveryTaskStore.java | 11 -- .../RetryablePostTransitionExecutor.java | 24 --- .../application/TransitionCommitResult.java | 24 --- .../arch/policy/book/domain/BookOrder.java | 54 ------- .../book/domain/DefaultOrderTransitions.java | 55 ------- .../arch/policy/book/domain/OrderEvent.java | 17 -- .../arch/policy/book/domain/OrderState.java | 26 --- .../policy/book/domain/OrderStateMachine.java | 72 --------- .../policy/book/domain/OrderTransition.java | 61 ------- .../book/domain/RequiredAttributeGuard.java | 16 -- .../book/domain/StateTransitionContext.java | 30 ---- .../policy/book/domain/TransitionAction.java | 5 - .../book/domain/TransitionExecution.java | 18 --- .../policy/book/domain/TransitionGuard.java | 5 - .../cache/DefaultCreateOrderResultCache.java | 54 ------- .../demo/DemoGdsBookingGateway.java | 18 --- .../job/OrderCreationOutboxScheduler.java | 15 -- .../job/OrderRecoveryScheduler.java | 13 -- .../repository/InMemoryBookOrderStore.java | 111 ------------- .../InMemoryFailedPostActionStore.java | 37 ----- .../InMemoryOrderWorkflowTaskStore.java | 17 -- .../repository/InMemoryRecoveryTaskStore.java | 45 ------ .../rpc/DubboBookOrderService.java | 29 ---- .../seata/SeataOrderCreationSaga.java | 37 ----- .../policy/common/book/BookOrderRequest.java | 27 ---- .../policy/common/book/BookOrderResponse.java | 20 --- .../policy/common/book/OrderEventRequest.java | 32 ---- .../common/config/BookOrderConfiguration.java | 133 --------------- .../common/config/SeataSagaConfiguration.java | 6 +- .../policy/order/cache/OrderCacheUtil.java | 47 ++++++ .../policy/order/cache/OrderLockUtil.java | 45 ++++++ .../order/model/InventoryTransaction.java | 23 +++ .../model/InventoryTransactionStatus.java | 25 +++ .../arch/policy/order/model/OrderRequest.java | 65 ++++++++ .../policy/order/model/OrderResponse.java | 30 ++++ .../arch/policy/order/model/OrderStatus.java | 25 +++ .../policy/order/model/SupplierOrder.java | 26 +++ .../order/model/SupplierOrderStatus.java | 28 ++++ .../policy/order/model/ThirdOrderResult.java | 14 ++ .../policy/order/model/ThirdOrderStatus.java | 13 ++ .../arch/policy/order/model/TradeOrder.java | 23 +++ .../service/InventoryTransactionService.java | 80 +++++++++ .../order/service/OrderCreationSaga.java | 9 ++ .../OrderCreationSagaStateServices.java | 74 +++++++++ .../order/service/OrderRecoveryService.java | 67 ++++++++ .../order/service/OrderRpcServiceImpl.java | 116 +++++++++++++ .../service/OrderTransactionService.java | 43 +++++ .../order/service/SeataOrderCreationSaga.java | 31 ++++ .../order/service/SupplierOrderService.java | 29 ++++ .../SupplierOrderTransactionService.java | 52 ++++++ .../statelang/book_order_creation_saga.json | 54 ------- .../statelang/order_creation_saga.json | 68 ++++++++ 79 files changed, 950 insertions(+), 1922 deletions(-) delete mode 100644 architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java create mode 100644 architecture/src/main/java/com/arch/policy/api/order/OrderRpcService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderState.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java delete mode 100644 architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java delete mode 100644 architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java delete mode 100644 architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java delete mode 100644 architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java delete mode 100644 architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java create mode 100644 architecture/src/main/java/com/arch/policy/order/cache/OrderCacheUtil.java create mode 100644 architecture/src/main/java/com/arch/policy/order/cache/OrderLockUtil.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/InventoryTransaction.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/InventoryTransactionStatus.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/OrderRequest.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/OrderResponse.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/OrderStatus.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/SupplierOrder.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/SupplierOrderStatus.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/ThirdOrderResult.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/ThirdOrderStatus.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/InventoryTransactionService.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/OrderCreationSaga.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/OrderRpcServiceImpl.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/SeataOrderCreationSaga.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/SupplierOrderService.java create mode 100644 architecture/src/main/java/com/arch/policy/order/service/SupplierOrderTransactionService.java delete mode 100644 architecture/src/main/resources/statelang/book_order_creation_saga.json create mode 100644 architecture/src/main/resources/statelang/order_creation_saga.json diff --git a/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java b/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java deleted file mode 100644 index ff75a2c..0000000 --- a/architecture/src/main/java/com/arch/policy/api/book/BookOrderRpcService.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.arch.policy.api.book; - -import com.arch.policy.common.book.BookOrderRequest; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.book.OrderEventRequest; - -public interface BookOrderRpcService { - - BookOrderResponse createOrder(BookOrderRequest request); - - BookOrderResponse fireEvent(OrderEventRequest request); -} diff --git a/architecture/src/main/java/com/arch/policy/api/order/OrderRpcService.java b/architecture/src/main/java/com/arch/policy/api/order/OrderRpcService.java new file mode 100644 index 0000000..2b2c840 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/api/order/OrderRpcService.java @@ -0,0 +1,14 @@ +package com.arch.policy.api.order; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.OrderResponse; + +/** + * @Author : haiyang.luo + * @Date : 2026/7/22 15:44 + * @Description : + */ +public interface OrderRpcService { + + OrderResponse createOrder(OrderRequest orderRequest); +} diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java deleted file mode 100644 index 6a9694f..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/BookOrderApplicationService.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; -import com.arch.policy.common.book.BookOrderRequest; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.lock.DistributedLock; -import com.arch.policy.common.lock.DistributedLockUnavailableException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigDecimal; -import java.util.UUID; - -public final class BookOrderApplicationService { - private static final Logger LOGGER = LoggerFactory.getLogger(BookOrderApplicationService.class); - private static final DistributedLock.LockHandle NO_OP_LOCK = () -> { }; - private static final long CREATE_LOCK_WAIT_MILLIS = 300L; - private static final String CREATE_ORDER_LOCK_KEY_PREFIX = "book:create:lock:"; - private static final long CREATE_RESULT_WAIT_MILLIS = 500L; - private static final long CREATE_RESULT_TTL_MILLIS = 600_000L; - private final BookOrderStore store; - private final OrderCreationOutboxPublisher outboxPublisher; - private final DistributedLock distributedLock; - private final CreateOrderResultCache resultCache; - - public BookOrderApplicationService(BookOrderStore store, - OrderCreationOutboxPublisher outboxPublisher, - DistributedLock distributedLock, - CreateOrderResultCache resultCache) { - this.store = store; - this.outboxPublisher = outboxPublisher; - this.distributedLock = distributedLock; - this.resultCache = resultCache; - } - - /** - * 创建订单并在本地事务提交后启动 GDS 占编 Saga。 - * - *

流程边界:

- *
    - *
  1. 使用 requestId 查询 Redis 幂等结果缓存,命中则直接返回。
  2. - *
  3. 获取 requestId 对应的 Redisson 短锁并进行缓存二次检查。
  4. - *
  5. 在同一本地事务中保存 CREATE 状态订单和 Saga 启动 Outbox。
  6. - *
  7. 事务提交后发布 Outbox,使用 orderNo 幂等启动 Seata Saga。
  8. - *
  9. Saga 根据 GDS 的 SUCCESS、FAIL、UNKNOWN 结果分别推进至 - * WAIT_PAY、CREATE_FAIL、VALIDATING。
  10. - *
  11. 重新读取订单并返回 Saga 执行后的最新状态。
  12. - *
- * - *

如果进程在订单提交后、Saga 启动前退出,定时发布器会重新投递未发布的 Outbox; - * Seata 根据持久化的状态机日志恢复已经启动的 Saga。

- * - * @param request 下单请求,requestId 是业务幂等键 - * @return 当前最新订单状态;同步 Saga 通常返回 WAIT_PAY、CREATE_FAIL 或 VALIDATING - * @throws IllegalArgumentException 请求参数不合法时抛出 - */ - public BookOrderResponse createOrder(BookOrderRequest request) { - validate(request); - - // 锁前只访问 Redis,不查询数据库,避免大量重复请求同时穿透到主库。 - BookOrderResponse cached = resultCache.get(request.getRequestId()); - if (cached != null) return cached; - - DistributedLock.LockHandle lock = acquireCreateOrderLock(request.getRequestId()); - if (lock == null) return responseOfCompetingRequest(request.getRequestId()); - - BookOrder order; - OrderCreationOutboxMessage outboxMessage; - try { - // Double check:前一个持锁请求可能已在等待期间完成创建并回填缓存。 - cached = resultCache.get(request.getRequestId()); - if (cached != null) return cached; - - // 只有锁持有者才能访问数据库;用于缓存丢失、过期及 Redis 降级场景。 - BookOrder existing = store.findByRequestId(request.getRequestId()); - if (existing != null) { - BookOrderResponse existingResponse = response(existing); - resultCache.put(request.getRequestId(), existingResponse, CREATE_RESULT_TTL_MILLIS); - return existingResponse; - } - - // 构造下单对象(模拟下单的内容) - order = BookOrder.create(newOrderNo(), request.getRequestId(), request.getCustomerId(), - request.getProductId(), request.getPromotionId(), request.getQuantity(), - request.getAmount()); - - // 数据库 request_id 唯一索引是最终幂等防线。 - outboxMessage = new OrderCreationOutboxMessage(order.getOrderNo(), - order.getPromotionId()); - LocalCreateResult localResult = store.createOrder(order, outboxMessage); - if (localResult.isDuplicateRequest()) { - BookOrderResponse duplicateResponse = response(localResult.getOrder()); - resultCache.put(request.getRequestId(), duplicateResponse, CREATE_RESULT_TTL_MILLIS); - return duplicateResponse; - } - resultCache.put(request.getRequestId(), response(order), CREATE_RESULT_TTL_MILLIS); - } finally { - lock.close(); - } - - // Redis 锁已释放;立即发布一次,崩溃遗漏由定时 Outbox 发布器补发。 - try { - outboxPublisher.publish(outboxMessage); - } catch (RuntimeException publishFailure) { - LOGGER.warn("Creation Outbox will be retried, orderNo={}", order.getOrderNo(), - publishFailure); - } - - // 同步 Saga 可能已经改变订单状态,因此返回前重新读取聚合快照。 - BookOrder latest = store.findByOrderNo(order.getOrderNo()); - BookOrderResponse latestResponse = response(latest == null ? order : latest); - resultCache.put(request.getRequestId(), latestResponse, CREATE_RESULT_TTL_MILLIS); - return latestResponse; - } - - private BookOrderResponse responseOfCompetingRequest(String requestId) { - BookOrderResponse cached = resultCache.await(requestId, CREATE_RESULT_WAIT_MILLIS); - if (cached != null) return cached; - throw new IllegalStateException("create order request is being processed: " + requestId); - } - - private DistributedLock.LockHandle acquireCreateOrderLock(String requestId) { - String key = CREATE_ORDER_LOCK_KEY_PREFIX + requestId; - try { - return distributedLock.tryAcquire(key, CREATE_LOCK_WAIT_MILLIS); - } catch (DistributedLockUnavailableException unavailable) { - // 数据库 request_id 唯一索引仍是最终幂等防线,Redis 故障时允许降级。 - LOGGER.warn("Create-order lock unavailable, fall back to database idempotency, key={}", - key, unavailable); - return NO_OP_LOCK; - } - } - - private static void validate(BookOrderRequest request) { - if (request == null) throw new IllegalArgumentException("request is required"); - if (isBlank(request.getRequestId())) throw new IllegalArgumentException("requestId is required"); - if (isBlank(request.getCustomerId())) throw new IllegalArgumentException("customerId is required"); - if (isBlank(request.getProductId())) throw new IllegalArgumentException("productId is required"); - if (request.getQuantity() <= 0) throw new IllegalArgumentException("quantity must be positive"); - if (request.getAmount() == null || request.getAmount().compareTo(BigDecimal.ZERO) < 0) { - throw new IllegalArgumentException("amount must not be negative"); - } - } - - private static String newOrderNo() { - return "BO" + UUID.randomUUID().toString().replace("-", "").substring(0, 20).toUpperCase(); - } - - private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } - private static BookOrderResponse response(BookOrder order) { - return new BookOrderResponse(order.getOrderNo(), order.getState().name(), order.getVersion()); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java deleted file mode 100644 index cf2bd4e..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/BookOrderStore.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; - -import java.util.List; - -public interface BookOrderStore { - BookOrder findByOrderNo(String orderNo); - - BookOrder findByRequestId(String requestId); - - BookOrder findByEventId(String eventId); - - LocalCreateResult createOrder(BookOrder order, OrderCreationOutboxMessage outboxMessage); - - List findUnpublishedCreationOutbox(int limit); - - void markCreationOutboxPublished(String messageId); - - TransitionCommitResult transit(BookOrder order, long expectedVersion, - OrderStateHistory history, OrderOutboxMessage outboxMessage); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java b/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java deleted file mode 100644 index 843c8f8..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/CompensationTaskService.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.arch.policy.book.application; - -import java.util.HashMap; -import java.util.Map; - -public final class CompensationTaskService { - private final RecoveryTaskStore taskStore; - - public CompensationTaskService(RecoveryTaskStore taskStore) { this.taskStore = taskStore; } - - public void returnPromotionStock(String orderNo, String promotionId) { - if (isBlank(promotionId)) return; - enqueue("PROMOTION:" + orderNo + ":" + promotionId, - RecoveryTask.RecoveryType.RETURN_PROMOTION_STOCK, orderNo, - params("promotionId", promotionId)); - } - - public void cancelPnr(String orderNo, String childOrderNo, String pnr) { - if (isBlank(pnr)) return; - Map parameters = params("childOrderNo", childOrderNo); - parameters.put("pnr", pnr); - enqueue("CANCEL_PNR:" + orderNo + ":" + childOrderNo + ":" + pnr, - RecoveryTask.RecoveryType.CANCEL_PNR, orderNo, parameters); - } - - public void refundPayment(String orderNo, String paymentNo) { - if (isBlank(paymentNo)) return; - enqueue("REFUND:" + orderNo + ":" + paymentNo, - RecoveryTask.RecoveryType.REFUND_PAYMENT, orderNo, - params("paymentNo", paymentNo)); - } - - public void validateGdsBooking(String orderNo) { - enqueue("VALIDATE_GDS:" + orderNo, - RecoveryTask.RecoveryType.VALIDATE_GDS_BOOKING, orderNo, - new HashMap()); - } - - private void enqueue(String uniqueKey, RecoveryTask.RecoveryType type, String orderNo, - Map parameters) { - taskStore.saveIfAbsent(new RecoveryTask(uniqueKey, type, orderNo, parameters)); - } - - private static Map params(String name, String value) { - Map params = new HashMap(); - params.put(name, value == null ? "" : value); - return params; - } - - private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java b/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java deleted file mode 100644 index db48c56..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/CreateOrderResultCache.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.common.book.BookOrderResponse; - -public interface CreateOrderResultCache { - BookOrderResponse get(String requestId); - - void put(String requestId, BookOrderResponse response, long ttlMillis); - - BookOrderResponse await(String requestId, long waitMillis); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java deleted file mode 100644 index cda115b..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/DirectOrderCreationSaga.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.arch.policy.book.application; - -public final class DirectOrderCreationSaga implements OrderCreationSaga { - private final OrderCreationSagaStateServices services; - - public DirectOrderCreationSaga(OrderCreationSagaStateServices services) { this.services = services; } - - @Override public SagaStartResult start(String orderNo, String promotionId) { - GdsBookingResult result; - try { - result = services.reservePnr(orderNo); - } catch (RuntimeException uncertainFailure) { - result = GdsBookingResult.unknown(uncertainFailure.getClass().getSimpleName()); - } - if (result.getStatus() == ExternalResultStatus.SUCCESS) { - services.markCreateSuccess(orderNo, result.getPnr()); - } else if (result.getStatus() == ExternalResultStatus.FAIL) { - services.markCreateFailure(orderNo, result.getErrorCode(), promotionId); - } else { - services.markCreateUnknown(orderNo, result.getErrorCode()); - } - return new SagaStartResult("DIRECT:" + orderNo, false); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java b/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java deleted file mode 100644 index 099da1a..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/ExternalResultStatus.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.arch.policy.book.application; - -public enum ExternalResultStatus { - SUCCESS, - FAIL, - UNKNOWN -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java deleted file mode 100644 index cc689f1..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingGateway.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.arch.policy.book.application; - -public interface GdsBookingGateway { - GdsBookingResult createPnr(String orderNo); - - GdsBookingResult queryPnr(String orderNo); - - void cancelPnr(String orderNo, String childOrderNo, String pnr); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java b/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java deleted file mode 100644 index 72b6d20..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/GdsBookingResult.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.arch.policy.book.application; - -public final class GdsBookingResult { - private final ExternalResultStatus status; - private final String pnr; - private final String errorCode; - - public GdsBookingResult(ExternalResultStatus status, String pnr, String errorCode) { - this.status = status; - this.pnr = pnr; - this.errorCode = errorCode; - } - - public static GdsBookingResult success(String pnr) { - return new GdsBookingResult(ExternalResultStatus.SUCCESS, pnr, null); - } - - public static GdsBookingResult fail(String errorCode) { - return new GdsBookingResult(ExternalResultStatus.FAIL, null, errorCode); - } - - public static GdsBookingResult unknown(String errorCode) { - return new GdsBookingResult(ExternalResultStatus.UNKNOWN, null, errorCode); - } - - public ExternalResultStatus getStatus() { return status; } - public String getPnr() { return pnr; } - public String getErrorCode() { return errorCode; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java b/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java deleted file mode 100644 index 8527498..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/GdsReconciliationService.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.arch.policy.book.application; - -public final class GdsReconciliationService implements OrderReconciler { - private final GdsBookingGateway gdsGateway; - private final OrderCreationSagaStateServices sagaStates; - - public GdsReconciliationService(GdsBookingGateway gdsGateway, - OrderCreationSagaStateServices sagaStates) { - this.gdsGateway = gdsGateway; - this.sagaStates = sagaStates; - } - - @Override public void reconcile(String orderNo) { - GdsBookingResult result = gdsGateway.queryPnr(orderNo); - if (result.getStatus() == ExternalResultStatus.SUCCESS) { - sagaStates.markGdsValidationSuccess(orderNo, result.getPnr()); - return; - } - if (result.getStatus() == ExternalResultStatus.FAIL) { - sagaStates.markGdsValidationFailure(orderNo, result.getErrorCode()); - return; - } - throw new IllegalStateException("GDS result is still unknown"); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java b/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java deleted file mode 100644 index dbeaadd..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/LocalCreateResult.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; - -public final class LocalCreateResult { - private final BookOrder order; - private final boolean duplicateRequest; - - private LocalCreateResult(BookOrder order, boolean duplicateRequest) { - this.order = order; - this.duplicateRequest = duplicateRequest; - } - - public static LocalCreateResult created(BookOrder order) { - return new LocalCreateResult(order, false); - } - - public static LocalCreateResult duplicate(BookOrder order) { - return new LocalCreateResult(order, true); - } - - public BookOrder getOrder() { return order; } - public boolean isDuplicateRequest() { return duplicateRequest; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java deleted file mode 100644 index cd52227..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxMessage.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.arch.policy.book.application; - -/** 与订单在同一本地事务中保存,用于可靠启动创建 Saga。 */ -public final class OrderCreationOutboxMessage { - private final String messageId; - private final String orderNo; - private final String promotionId; - private boolean published; - - public OrderCreationOutboxMessage(String orderNo, String promotionId) { - this.messageId = "START_ORDER_CREATION:" + orderNo; - this.orderNo = orderNo; - this.promotionId = promotionId; - } - - public String getMessageId() { return messageId; } - public String getOrderNo() { return orderNo; } - public String getPromotionId() { return promotionId; } - public boolean isPublished() { return published; } - public void markPublished() { published = true; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java deleted file mode 100644 index a1c5ec7..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationOutboxPublisher.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.arch.policy.book.application; - -import java.util.List; - -/** 以至少一次语义发布 Outbox;OrderCreationSaga 必须使用 orderNo 幂等启动。 */ -public final class OrderCreationOutboxPublisher { - private final BookOrderStore store; - private final OrderCreationSaga creationSaga; - - public OrderCreationOutboxPublisher(BookOrderStore store, OrderCreationSaga creationSaga) { - this.store = store; - this.creationSaga = creationSaga; - } - - public void publish(OrderCreationOutboxMessage message) { - if (message.isPublished()) return; - creationSaga.start(message.getOrderNo(), message.getPromotionId()); - store.markCreationOutboxPublished(message.getMessageId()); - } - - public void publishPending(int limit) { - List messages = store.findUnpublishedCreationOutbox(limit); - for (OrderCreationOutboxMessage message : messages) publish(message); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java deleted file mode 100644 index 9f9637c..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSaga.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.arch.policy.book.application; - -public interface OrderCreationSaga { - - /** 使用 orderNo 作为业务键幂等启动;重复调用不得重复执行外部业务动作。 */ - SagaStartResult start(String orderNo, String promotionId); - - final class SagaStartResult { - private final String instanceId; - private final boolean running; - - public SagaStartResult(String instanceId, boolean running) { - this.instanceId = instanceId; - this.running = running; - } - - public String getInstanceId() { return instanceId; } - public boolean isRunning() { return running; } - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java b/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java deleted file mode 100644 index 366c613..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderCreationSagaStateServices.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; -import com.arch.policy.common.book.OrderEventRequest; - -import java.util.Collections; - -public final class OrderCreationSagaStateServices { - private final BookOrderStore store; - private final GdsBookingGateway gdsGateway; - private final OrderEventApplicationService eventService; - private final CompensationTaskService compensationService; - private final OrderWorkflowTaskStore workflowTaskStore; - - public OrderCreationSagaStateServices(BookOrderStore store, GdsBookingGateway gdsGateway, - OrderEventApplicationService eventService, - CompensationTaskService compensationService, - OrderWorkflowTaskStore workflowTaskStore) { - this.store = store; - this.gdsGateway = gdsGateway; - this.eventService = eventService; - this.compensationService = compensationService; - this.workflowTaskStore = workflowTaskStore; - } - - public GdsBookingResult reservePnr(String orderNo) { return gdsGateway.createPnr(orderNo); } - - public boolean markCreateSuccess(String orderNo, String pnr) { - fire(orderNo, orderNo + ":CREATE_SUCCESS", "CREATE_SUCCEEDED", "gds-booking-saga", - Collections.singletonMap("bookingReference", value(pnr))); - workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("WAIT_PAYMENT:" + orderNo, - orderNo, OrderWorkflowTask.TaskType.WAIT_PAYMENT)); - return true; - } - - public boolean markCreateFailure(String orderNo, String errorCode, String promotionId) { - fire(orderNo, orderNo + ":CREATE_FAIL", "CREATE_FAILED", "gds-booking-saga", - Collections.singletonMap("failureCode", value(errorCode))); - compensationService.returnPromotionStock(orderNo, value(promotionId)); - return true; - } - - public boolean markCreateUnknown(String orderNo, String errorCode) { - fire(orderNo, orderNo + ":CREATE_UNKNOWN", "START_VALIDATE", "gds-booking-saga", - Collections.singletonMap("failureCode", value(errorCode))); - workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("MANUAL_GDS_VALIDATION:" + orderNo, - orderNo, OrderWorkflowTask.TaskType.MANUAL_GDS_VALIDATION)); - compensationService.validateGdsBooking(orderNo); - return true; - } - - public boolean enqueueCancelPnr(String orderNo, String childOrderNo, String pnr) { - compensationService.cancelPnr(orderNo, childOrderNo, pnr); - return true; - } - - public void markGdsValidationSuccess(String orderNo, String pnr) { - fire(orderNo, orderNo + ":RECONCILE_SUCCESS", "GDS_BOOKING_CONFIRMED", - "gds-reconciliation-job", Collections.singletonMap("bookingReference", value(pnr))); - workflowTaskStore.saveIfAbsent(new OrderWorkflowTask("WAIT_PAYMENT:" + orderNo, - orderNo, OrderWorkflowTask.TaskType.WAIT_PAYMENT)); - } - - public void markGdsValidationFailure(String orderNo, String errorCode) { - BookOrder order = store.findByOrderNo(orderNo); - fire(orderNo, orderNo + ":RECONCILE_FAIL", "GDS_BOOKING_REJECTED", - "gds-reconciliation-job", Collections.singletonMap("failureCode", value(errorCode))); - compensationService.returnPromotionStock(orderNo, value(order.getPromotionId())); - } - - private void fire(String orderNo, String eventId, String event, String operator, - java.util.Map attributes) { - BookOrder order = store.findByOrderNo(orderNo); - OrderEventRequest request = new OrderEventRequest(); - request.setOrderNo(orderNo); - request.setEventId(eventId); - request.setEvent(event); - request.setOperator(operator); - request.setExpectedVersion(order.getVersion()); - request.setAttributes(attributes); - eventService.fireEvent(request); - } - - private static String value(String value) { return value == null ? "" : value; } - -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java b/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java deleted file mode 100644 index 4dc9c52..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderEventApplicationService.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; -import com.arch.policy.book.domain.OrderEvent; -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.domain.StateTransitionContext; -import com.arch.policy.book.domain.TransitionExecution; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.book.OrderEventRequest; - -public final class OrderEventApplicationService { - private static final long CREATE_RESULT_TTL_MILLIS = 600_000L; - private final BookOrderStore store; - private final OrderStateMachine stateMachine; - private final PostTransitionExecutor postTransitionExecutor; - private final CompensationTaskService compensationTaskService; - private final CreateOrderResultCache resultCache; - - public OrderEventApplicationService(BookOrderStore store, OrderStateMachine stateMachine, - PostTransitionExecutor postTransitionExecutor, - CompensationTaskService compensationTaskService, - CreateOrderResultCache resultCache) { - this.store = store; - this.stateMachine = stateMachine; - this.postTransitionExecutor = postTransitionExecutor; - this.compensationTaskService = compensationTaskService; - this.resultCache = resultCache; - } - - public BookOrderResponse fireEvent(OrderEventRequest request) { - validate(request); - BookOrder processed = store.findByEventId(request.getEventId()); - if (processed != null) return response(processed); - BookOrder order = store.findByOrderNo(request.getOrderNo()); - if (order == null) throw new IllegalArgumentException("order not found: " + request.getOrderNo()); - if (order.getVersion() != request.getExpectedVersion()) { - throw new IllegalStateException("stale order version: " + request.getExpectedVersion()); - } - OrderEvent event = parseEvent(request.getEvent()); - TransitionExecution execution = stateMachine.fire(new StateTransitionContext( - request.getEventId(), event, order, request.getOperator(), request.getAttributes())); - TransitionCommitResult result = store.transit(order, execution.getPreviousVersion(), - history(execution), outbox(execution)); - if (!result.isDuplicateEvent()) { - enqueueCompensation(execution); - postTransitionExecutor.execute(stateMachine, execution); - } - BookOrderResponse response = response(result.getOrder()); - resultCache.put(result.getOrder().getRequestId(), response, CREATE_RESULT_TTL_MILLIS); - return response; - } - - private void enqueueCompensation(TransitionExecution execution) { - StateTransitionContext context = execution.getContext(); - if (context.getEvent() == OrderEvent.CANCEL && !isBlank(context.attribute("pnr"))) { - compensationTaskService.cancelPnr(context.getOrder().getOrderNo(), - value(context.attribute("childOrderNo")), context.attribute("pnr")); - } - if (context.getEvent() == OrderEvent.BOOK_FAILED && !isBlank(context.attribute("paymentNo"))) { - compensationTaskService.refundPayment(context.getOrder().getOrderNo(), - context.attribute("paymentNo")); - } - } - - private static OrderStateHistory history(TransitionExecution execution) { - return new OrderStateHistory(execution.getContext().getEventId(), - execution.getContext().getOrder().getOrderNo(), execution.getTransition().getFrom(), - execution.getTransition().getEvent(), execution.getTransition().getTo(), - execution.getContext().getOperator(), System.currentTimeMillis()); - } - - private static OrderOutboxMessage outbox(TransitionExecution execution) { - BookOrder order = execution.getContext().getOrder(); - return new OrderOutboxMessage(execution.getContext().getEventId(), order.getOrderNo(), - execution.getContext().getEvent(), order.getState(), order.getVersion()); - } - - private static void validate(OrderEventRequest request) { - if (request == null || isBlank(request.getEventId()) || isBlank(request.getOrderNo()) - || isBlank(request.getEvent()) || isBlank(request.getOperator())) { - throw new IllegalArgumentException("eventId, orderNo, event and operator are required"); - } - } - - private static OrderEvent parseEvent(String event) { - try { return OrderEvent.valueOf(event.trim().toUpperCase()); } - catch (IllegalArgumentException failure) { - throw new IllegalArgumentException("unknown order event: " + event, failure); - } - } - - private static boolean isBlank(String value) { return value == null || value.trim().isEmpty(); } - private static String value(String value) { return value == null ? "" : value; } - private static BookOrderResponse response(BookOrder order) { - return new BookOrderResponse(order.getOrderNo(), order.getState().name(), order.getVersion()); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java b/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java deleted file mode 100644 index 9e0b4c3..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderOutboxMessage.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.OrderEvent; -import com.arch.policy.book.domain.OrderState; - -public final class OrderOutboxMessage { - private final String eventId; - private final String orderNo; - private final OrderEvent event; - private final OrderState state; - private final long orderVersion; - private boolean published; - - public OrderOutboxMessage(String eventId, String orderNo, OrderEvent event, - OrderState state, long orderVersion) { - this.eventId = eventId; - this.orderNo = orderNo; - this.event = event; - this.state = state; - this.orderVersion = orderVersion; - } - - public String getEventId() { return eventId; } - public String getOrderNo() { return orderNo; } - public OrderEvent getEvent() { return event; } - public OrderState getState() { return state; } - public long getOrderVersion() { return orderVersion; } - public boolean isPublished() { return published; } - public void markPublished() { published = true; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java b/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java deleted file mode 100644 index 2420f70..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderReconciler.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.application; - -public interface OrderReconciler { - void reconcile(String orderNo); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java b/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java deleted file mode 100644 index 38e8fab..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderStateHistory.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.OrderEvent; -import com.arch.policy.book.domain.OrderState; - -public final class OrderStateHistory { - private final String eventId; - private final String orderNo; - private final OrderState from; - private final OrderEvent event; - private final OrderState to; - private final String operator; - private final long occurredAtMillis; - - public OrderStateHistory(String eventId, String orderNo, OrderState from, OrderEvent event, - OrderState to, String operator, long occurredAtMillis) { - this.eventId = eventId; - this.orderNo = orderNo; - this.from = from; - this.event = event; - this.to = to; - this.operator = operator; - this.occurredAtMillis = occurredAtMillis; - } - - public String getEventId() { return eventId; } - public String getOrderNo() { return orderNo; } - public OrderState getFrom() { return from; } - public OrderEvent getEvent() { return event; } - public OrderState getTo() { return to; } - public String getOperator() { return operator; } - public long getOccurredAtMillis() { return occurredAtMillis; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java deleted file mode 100644 index de6d653..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTask.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.arch.policy.book.application; - -public final class OrderWorkflowTask { - private final String uniqueKey; - private final String orderNo; - private final TaskType type; - - public OrderWorkflowTask(String uniqueKey, String orderNo, TaskType type) { - this.uniqueKey = uniqueKey; - this.orderNo = orderNo; - this.type = type; - } - - public String getUniqueKey() { return uniqueKey; } - public String getOrderNo() { return orderNo; } - public TaskType getType() { return type; } - - public enum TaskType { WAIT_PAYMENT, MANUAL_GDS_VALIDATION } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java b/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java deleted file mode 100644 index 88fb2d5..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/OrderWorkflowTaskStore.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.application; - -public interface OrderWorkflowTaskStore { - void saveIfAbsent(OrderWorkflowTask task); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java b/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java deleted file mode 100644 index 6c828b2..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/PaymentGateway.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.application; - -public interface PaymentGateway { - void refund(String orderNo, String paymentNo); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java b/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java deleted file mode 100644 index 74e5d54..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/PostTransitionExecutor.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.domain.TransitionExecution; - -public interface PostTransitionExecutor { - void execute(OrderStateMachine stateMachine, TransitionExecution execution); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java b/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java deleted file mode 100644 index 23a9132..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/PromotionStockGateway.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.application; - -public interface PromotionStockGateway { - void returnStock(String orderNo, String promotionId); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java deleted file mode 100644 index 0989144..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTask.java +++ /dev/null @@ -1,66 +0,0 @@ -package com.arch.policy.book.application; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public final class RecoveryTask { - private final String uniqueKey; - private final RecoveryType type; - private final String orderNo; - private final Map parameters; - private RecoveryStatus status; - private int attempts; - private long nextAttemptAtMillis; - private String lastError; - - public RecoveryTask(String uniqueKey, RecoveryType type, String orderNo, - Map parameters) { - this.uniqueKey = uniqueKey; - this.type = type; - this.orderNo = orderNo; - this.parameters = parameters == null ? Collections.emptyMap() - : Collections.unmodifiableMap(new HashMap(parameters)); - this.status = RecoveryStatus.PENDING; - } - - public RecoveryTask copy() { - RecoveryTask copy = new RecoveryTask(uniqueKey, type, orderNo, parameters); - copy.status = status; - copy.attempts = attempts; - copy.nextAttemptAtMillis = nextAttemptAtMillis; - copy.lastError = lastError; - return copy; - } - - public void markSucceeded() { status = RecoveryStatus.SUCCEEDED; lastError = null; } - public void markRetry(long nextAttemptAtMillis, String error) { - attempts++; - status = RecoveryStatus.RETRYING; - this.nextAttemptAtMillis = nextAttemptAtMillis; - lastError = error; - } - public void markManualRequired(String error) { - attempts++; - status = RecoveryStatus.MANUAL_REQUIRED; - lastError = error; - } - - public String getUniqueKey() { return uniqueKey; } - public RecoveryType getType() { return type; } - public String getOrderNo() { return orderNo; } - public Map getParameters() { return parameters; } - public RecoveryStatus getStatus() { return status; } - public int getAttempts() { return attempts; } - public long getNextAttemptAtMillis() { return nextAttemptAtMillis; } - public String getLastError() { return lastError; } - - public enum RecoveryType { - RETURN_PROMOTION_STOCK, - CANCEL_PNR, - REFUND_PAYMENT, - VALIDATE_GDS_BOOKING - } - - public enum RecoveryStatus { PENDING, RETRYING, SUCCEEDED, MANUAL_REQUIRED } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java deleted file mode 100644 index c010fda..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskProcessor.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.arch.policy.book.application; - -import java.util.List; - -public final class RecoveryTaskProcessor { - private static final int MAX_ATTEMPTS = 5; - private final RecoveryTaskStore taskStore; - private final PromotionStockGateway promotionStockGateway; - private final GdsBookingGateway gdsGateway; - private final PaymentGateway paymentGateway; - private final OrderReconciler reconciliationService; - - public RecoveryTaskProcessor(RecoveryTaskStore taskStore, - PromotionStockGateway promotionStockGateway, - GdsBookingGateway gdsGateway, - PaymentGateway paymentGateway, - OrderReconciler reconciliationService) { - this.taskStore = taskStore; - this.promotionStockGateway = promotionStockGateway; - this.gdsGateway = gdsGateway; - this.paymentGateway = paymentGateway; - this.reconciliationService = reconciliationService; - } - - public void processBatch(long nowMillis, int limit) { - List tasks = taskStore.findExecutable(nowMillis, limit); - for (RecoveryTask task : tasks) process(task, nowMillis); - } - - private void process(RecoveryTask task, long nowMillis) { - try { - execute(task); - task.markSucceeded(); - } catch (RuntimeException failure) { - if (task.getAttempts() + 1 >= MAX_ATTEMPTS) { - task.markManualRequired(failure.getMessage()); - } else { - task.markRetry(nowMillis + retryDelayMillis(task.getAttempts()), failure.getMessage()); - } - } - taskStore.save(task); - } - - private void execute(RecoveryTask task) { - switch (task.getType()) { - case RETURN_PROMOTION_STOCK: - promotionStockGateway.returnStock(task.getOrderNo(), task.getParameters().get("promotionId")); - return; - case CANCEL_PNR: - gdsGateway.cancelPnr(task.getOrderNo(), task.getParameters().get("childOrderNo"), - task.getParameters().get("pnr")); - return; - case REFUND_PAYMENT: - paymentGateway.refund(task.getOrderNo(), task.getParameters().get("paymentNo")); - return; - case VALIDATE_GDS_BOOKING: - reconciliationService.reconcile(task.getOrderNo()); - return; - default: - throw new IllegalStateException("unsupported recovery type: " + task.getType()); - } - } - - private static long retryDelayMillis(int attempts) { - return Math.min(60_000L, 1_000L << Math.min(attempts, 6)); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java b/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java deleted file mode 100644 index b8ac806..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/RecoveryTaskStore.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.arch.policy.book.application; - -import java.util.List; - -public interface RecoveryTaskStore { - RecoveryTask saveIfAbsent(RecoveryTask task); - - List findExecutable(long nowMillis, int limit); - - void save(RecoveryTask task); -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java b/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java deleted file mode 100644 index 3043b16..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/RetryablePostTransitionExecutor.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.domain.TransitionExecution; - -public final class RetryablePostTransitionExecutor implements PostTransitionExecutor { - private final FailedPostActionStore failedActionStore; - - public RetryablePostTransitionExecutor(FailedPostActionStore failedActionStore) { - this.failedActionStore = failedActionStore; - } - - @Override public void execute(OrderStateMachine stateMachine, TransitionExecution execution) { - try { - stateMachine.afterCommit(execution); - } catch (RuntimeException failure) { - failedActionStore.record(execution, failure); - } - } - - public interface FailedPostActionStore { - void record(TransitionExecution execution, RuntimeException failure); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java b/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java deleted file mode 100644 index 6564e33..0000000 --- a/architecture/src/main/java/com/arch/policy/book/application/TransitionCommitResult.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.arch.policy.book.application; - -import com.arch.policy.book.domain.BookOrder; - -public final class TransitionCommitResult { - private final BookOrder order; - private final boolean duplicateEvent; - - private TransitionCommitResult(BookOrder order, boolean duplicateEvent) { - this.order = order; - this.duplicateEvent = duplicateEvent; - } - - public static TransitionCommitResult committed(BookOrder order) { - return new TransitionCommitResult(order, false); - } - - public static TransitionCommitResult duplicate(BookOrder order) { - return new TransitionCommitResult(order, true); - } - - public BookOrder getOrder() { return order; } - public boolean isDuplicateEvent() { return duplicateEvent; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java b/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java deleted file mode 100644 index 6507001..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/BookOrder.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.arch.policy.book.domain; - -import java.math.BigDecimal; - -public final class BookOrder { - private final String orderNo; - private final String requestId; - private final String customerId; - private final String productId; - private final String promotionId; - private final int quantity; - private final BigDecimal amount; - private OrderState state; - private long version; - - private BookOrder(String orderNo, String requestId, String customerId, String productId, - String promotionId, int quantity, BigDecimal amount, OrderState state, long version) { - this.orderNo = orderNo; - this.requestId = requestId; - this.customerId = customerId; - this.productId = productId; - this.promotionId = promotionId; - this.quantity = quantity; - this.amount = amount; - this.state = state; - this.version = version; - } - - public static BookOrder create(String orderNo, String requestId, String customerId, - String productId, String promotionId, int quantity, BigDecimal amount) { - return new BookOrder(orderNo, requestId, customerId, productId, promotionId, quantity, amount, - OrderState.CREATE, 0L); - } - - public BookOrder copy() { - return new BookOrder(orderNo, requestId, customerId, productId, promotionId, - quantity, amount, state, version); - } - - void applyState(OrderState target) { - state = target; - version++; - } - - public String getOrderNo() { return orderNo; } - public String getRequestId() { return requestId; } - public String getCustomerId() { return customerId; } - public String getProductId() { return productId; } - public String getPromotionId() { return promotionId; } - public int getQuantity() { return quantity; } - public BigDecimal getAmount() { return amount; } - public OrderState getState() { return state; } - public long getVersion() { return version; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java b/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java deleted file mode 100644 index 2d51f92..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/DefaultOrderTransitions.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.arch.policy.book.domain; - -import java.util.ArrayList; -import java.util.List; - -public final class DefaultOrderTransitions { - private DefaultOrderTransitions() {} - - public static List definitions() { - List transitions = new ArrayList(); - add(transitions, OrderState.CREATE, OrderEvent.CREATE_SUCCEEDED, OrderState.WAIT_PAY); - add(transitions, OrderState.CREATE, OrderEvent.CREATE_FAILED, OrderState.CREATE_FAIL); - add(transitions, OrderState.CREATE, OrderEvent.START_VALIDATE, OrderState.VALIDATING); - add(transitions, OrderState.CREATE, OrderEvent.CANCEL, OrderState.CANCEL); - transitions.add(OrderTransition - .from(OrderState.WAIT_PAY, OrderEvent.PAY_SUCCEEDED, OrderState.BOOKING) - .guard(new RequiredAttributeGuard("paymentNo")).build()); - add(transitions, OrderState.WAIT_PAY, OrderEvent.CANCEL, OrderState.CANCEL); - transitions.add(OrderTransition - .from(OrderState.BOOKING, OrderEvent.BOOK_SUCCEEDED, OrderState.BOOKED) - .guard(new RequiredAttributeGuard("bookingReference")).build()); - transitions.add(OrderTransition - .from(OrderState.BOOKING, OrderEvent.BOOK_FAILED, OrderState.BOOK_FAIL) - .guard(new RequiredAttributeGuard("failureCode")).build()); - add(transitions, OrderState.BOOKING, OrderEvent.START_VALIDATE, OrderState.VALIDATING); - add(transitions, OrderState.BOOK_FAIL, OrderEvent.START_VALIDATE, OrderState.VALIDATING); - add(transitions, OrderState.BOOK_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); - add(transitions, OrderState.BOOK_FAIL, OrderEvent.DELETE, OrderState.DELETED); - add(transitions, OrderState.BOOKED, OrderEvent.START_VALIDATE, OrderState.VALIDATING); - add(transitions, OrderState.BOOKED, OrderEvent.CANCEL, OrderState.CANCEL); - transitions.add(OrderTransition - .from(OrderState.BOOKED, OrderEvent.REFUND_SUCCEEDED, OrderState.REFUNDED) - .guard(new RequiredAttributeGuard("refundNo")).build()); - add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); - add(transitions, OrderState.VALIDATING, OrderEvent.VALIDATE_FAILED, OrderState.VALIDATE_FAIL); - transitions.add(OrderTransition - .from(OrderState.VALIDATING, OrderEvent.GDS_BOOKING_CONFIRMED, OrderState.WAIT_PAY) - .guard(new RequiredAttributeGuard("bookingReference")).build()); - transitions.add(OrderTransition - .from(OrderState.VALIDATING, OrderEvent.GDS_BOOKING_REJECTED, OrderState.CREATE_FAIL) - .guard(new RequiredAttributeGuard("failureCode")).build()); - add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.VALIDATE_SUCCEEDED, OrderState.BOOKED); - add(transitions, OrderState.VALIDATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); - add(transitions, OrderState.CREATE_FAIL, OrderEvent.CANCEL, OrderState.CANCEL); - add(transitions, OrderState.CREATE_FAIL, OrderEvent.DELETE, OrderState.DELETED); - add(transitions, OrderState.CANCEL, OrderEvent.DELETE, OrderState.DELETED); - add(transitions, OrderState.REFUNDED, OrderEvent.DELETE, OrderState.DELETED); - return transitions; - } - - private static void add(List transitions, OrderState from, - OrderEvent event, OrderState to) { - transitions.add(OrderTransition.from(from, event, to).build()); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java deleted file mode 100644 index 9bd36b7..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/OrderEvent.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.arch.policy.book.domain; - -public enum OrderEvent { - CREATE_SUCCEEDED, - CREATE_FAILED, - PAY_SUCCEEDED, - BOOK_SUCCEEDED, - BOOK_FAILED, - START_VALIDATE, - VALIDATE_SUCCEEDED, - VALIDATE_FAILED, - GDS_BOOKING_CONFIRMED, - GDS_BOOKING_REJECTED, - CANCEL, - REFUND_SUCCEEDED, - DELETE -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java deleted file mode 100644 index a747609..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/OrderState.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.arch.policy.book.domain; - -public enum OrderState { - CREATE(0, "创单"), - WAIT_PAY(1, "待支付"), - CREATE_FAIL(2, "创单失败"), - CANCEL(3, "已取消"), - BOOKING(4, "预定中"), - BOOKED(5, "已预定"), - BOOK_FAIL(6, "预定失败"), - DELETED(7, "已删除"), - REFUNDED(8, "已退订"), - VALIDATING(10, "验真中"), - VALIDATE_FAIL(11, "验真失败"); - - private final int code; - private final String description; - - OrderState(int code, String description) { - this.code = code; - this.description = description; - } - - public int getCode() { return code; } - public String getDescription() { return description; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java deleted file mode 100644 index 7d5b19d..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/OrderStateMachine.java +++ /dev/null @@ -1,72 +0,0 @@ -package com.arch.policy.book.domain; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public final class OrderStateMachine { - private final Map transitions; - - public OrderStateMachine(List definitions) { - Map registry = new HashMap(); - for (OrderTransition transition : definitions) { - TransitionKey key = new TransitionKey(transition.getFrom(), transition.getEvent()); - if (registry.put(key, transition) != null) { - throw new IllegalStateException("duplicate order transition: " + key); - } - } - transitions = Collections.unmodifiableMap(registry); - } - - public TransitionExecution fire(StateTransitionContext context) { - OrderTransition transition = transitions.get( - new TransitionKey(context.getOrder().getState(), context.getEvent())); - if (transition == null) { - throw new IllegalStateException("unsupported order event " + context.getEvent() - + " in state " + context.getOrder().getState()); - } - executeGuards(transition, context); - executeActions(transition.getPreActions(), context); - long previousVersion = context.getOrder().getVersion(); - context.getOrder().applyState(transition.getTo()); - return new TransitionExecution(transition, context, previousVersion); - } - - public void afterCommit(TransitionExecution execution) { - executeActions(execution.getTransition().getPostActions(), execution.getContext()); - } - - public OrderState targetOf(OrderState state, OrderEvent event) { - OrderTransition transition = transitions.get(new TransitionKey(state, event)); - return transition == null ? null : transition.getTo(); - } - - private static void executeGuards(OrderTransition transition, StateTransitionContext context) { - for (TransitionGuard guard : transition.getGuards()) guard.check(context); - } - - private static void executeActions(List actions, StateTransitionContext context) { - for (TransitionAction action : actions) action.execute(context); - } - - private static final class TransitionKey { - private final OrderState state; - private final OrderEvent event; - - private TransitionKey(OrderState state, OrderEvent event) { - this.state = state; - this.event = event; - } - - @Override public boolean equals(Object other) { - if (this == other) return true; - if (!(other instanceof TransitionKey)) return false; - TransitionKey that = (TransitionKey) other; - return state == that.state && event == that.event; - } - - @Override public int hashCode() { return 31 * state.hashCode() + event.hashCode(); } - @Override public String toString() { return state + " + " + event; } - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java b/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java deleted file mode 100644 index 4ae61df..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/OrderTransition.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.arch.policy.book.domain; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public final class OrderTransition { - private final OrderState from; - private final OrderEvent event; - private final OrderState to; - private final List guards; - private final List preActions; - private final List postActions; - - private OrderTransition(Builder builder) { - from = builder.from; - event = builder.event; - to = builder.to; - guards = immutableCopy(builder.guards); - preActions = immutableCopy(builder.preActions); - postActions = immutableCopy(builder.postActions); - } - - public static Builder from(OrderState from, OrderEvent event, OrderState to) { - return new Builder(from, event, to); - } - - public OrderState getFrom() { return from; } - public OrderEvent getEvent() { return event; } - public OrderState getTo() { return to; } - public List getGuards() { return guards; } - public List getPreActions() { return preActions; } - public List getPostActions() { return postActions; } - - private static List immutableCopy(List values) { - return Collections.unmodifiableList(new ArrayList(values)); - } - - public static final class Builder { - private final OrderState from; - private final OrderEvent event; - private final OrderState to; - private final List guards = new ArrayList(); - private final List preActions = new ArrayList(); - private final List postActions = new ArrayList(); - - private Builder(OrderState from, OrderEvent event, OrderState to) { - if (from == null || event == null || to == null) { - throw new IllegalArgumentException("from, event and to are required"); - } - this.from = from; - this.event = event; - this.to = to; - } - - public Builder guard(TransitionGuard guard) { guards.add(guard); return this; } - public Builder before(TransitionAction action) { preActions.add(action); return this; } - public Builder after(TransitionAction action) { postActions.add(action); return this; } - public OrderTransition build() { return new OrderTransition(this); } - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java b/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java deleted file mode 100644 index b636a00..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/RequiredAttributeGuard.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.arch.policy.book.domain; - -public final class RequiredAttributeGuard implements TransitionGuard { - private final String attributeName; - - public RequiredAttributeGuard(String attributeName) { - this.attributeName = attributeName; - } - - @Override public void check(StateTransitionContext context) { - String value = context.attribute(attributeName); - if (value == null || value.trim().isEmpty()) { - throw new IllegalStateException(attributeName + " is required for event " + context.getEvent()); - } - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java b/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java deleted file mode 100644 index 3269260..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/StateTransitionContext.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.arch.policy.book.domain; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public final class StateTransitionContext { - private final String eventId; - private final OrderEvent event; - private final BookOrder order; - private final String operator; - private final Map attributes; - - public StateTransitionContext(String eventId, OrderEvent event, BookOrder order, - String operator, Map attributes) { - this.eventId = eventId; - this.event = event; - this.order = order; - this.operator = operator; - this.attributes = attributes == null ? Collections.emptyMap() - : Collections.unmodifiableMap(new HashMap(attributes)); - } - - public String getEventId() { return eventId; } - public OrderEvent getEvent() { return event; } - public BookOrder getOrder() { return order; } - public String getOperator() { return operator; } - public Map getAttributes() { return attributes; } - public String attribute(String name) { return attributes.get(name); } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java deleted file mode 100644 index a035155..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/TransitionAction.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.domain; - -public interface TransitionAction { - void execute(StateTransitionContext context); -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java deleted file mode 100644 index 1e4a0c8..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/TransitionExecution.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.arch.policy.book.domain; - -public final class TransitionExecution { - private final OrderTransition transition; - private final StateTransitionContext context; - private final long previousVersion; - - public TransitionExecution(OrderTransition transition, StateTransitionContext context, - long previousVersion) { - this.transition = transition; - this.context = context; - this.previousVersion = previousVersion; - } - - public OrderTransition getTransition() { return transition; } - public StateTransitionContext getContext() { return context; } - public long getPreviousVersion() { return previousVersion; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java b/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java deleted file mode 100644 index 5c88ebe..0000000 --- a/architecture/src/main/java/com/arch/policy/book/domain/TransitionGuard.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.arch.policy.book.domain; - -public interface TransitionGuard { - void check(StateTransitionContext context); -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java deleted file mode 100644 index fd6bf92..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/cache/DefaultCreateOrderResultCache.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.arch.policy.book.infrastructure.cache; - -import com.arch.policy.book.application.CreateOrderResultCache; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.redis.RedisClient; -import com.arch.policy.common.redis.RedisClientException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.locks.LockSupport; - -public final class DefaultCreateOrderResultCache implements CreateOrderResultCache { - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultCreateOrderResultCache.class); - private static final String KEY_PREFIX = "book:create:result:"; - private static final long POLL_INTERVAL_NANOS = 20_000_000L; - private final RedisClient redisClient; - - public DefaultCreateOrderResultCache(RedisClient redisClient) { - this.redisClient = redisClient; - } - - @Override public BookOrderResponse get(String requestId) { - try { - return redisClient.get(key(requestId)); - } catch (RedisClientException redisFailure) { - LOGGER.warn("Create-order result cache unavailable, requestId={}", requestId, - redisFailure); - return null; - } - } - - @Override public void put(String requestId, BookOrderResponse response, long ttlMillis) { - try { - redisClient.set(key(requestId), response, ttlMillis); - } catch (RedisClientException redisFailure) { - // 缓存写失败不回滚已经提交的订单,后续请求仍可在锁内查询数据库回填。 - LOGGER.warn("Failed to cache create-order result, requestId={}", requestId, - redisFailure); - } - } - - @Override public BookOrderResponse await(String requestId, long waitMillis) { - long deadlineNanos = System.nanoTime() + waitMillis * 1_000_000L; - do { - BookOrderResponse response = get(requestId); - if (response != null) return response; - if (Thread.currentThread().isInterrupted()) return null; - LockSupport.parkNanos(POLL_INTERVAL_NANOS); - } while (System.nanoTime() < deadlineNanos); - return get(requestId); - } - - private static String key(String requestId) { return KEY_PREFIX + requestId; } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java deleted file mode 100644 index d965d60..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/demo/DemoGdsBookingGateway.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.arch.policy.book.infrastructure.demo; - -import com.arch.policy.book.application.GdsBookingGateway; -import com.arch.policy.book.application.GdsBookingResult; - -public final class DemoGdsBookingGateway implements GdsBookingGateway { - @Override public GdsBookingResult createPnr(String orderNo) { - return GdsBookingResult.success("PNR-" + orderNo.substring(Math.max(0, orderNo.length() - 6))); - } - - @Override public GdsBookingResult queryPnr(String orderNo) { - return createPnr(orderNo); - } - - @Override public void cancelPnr(String orderNo, String childOrderNo, String pnr) { - // Demo adapter: production uses a GDS RPC client with the same idempotency key. - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java deleted file mode 100644 index 2094263..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderCreationOutboxScheduler.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.arch.policy.book.infrastructure.job; - -import com.arch.policy.book.application.OrderCreationOutboxPublisher; -import org.springframework.scheduling.annotation.Scheduled; - -public final class OrderCreationOutboxScheduler { - private final OrderCreationOutboxPublisher publisher; - - public OrderCreationOutboxScheduler(OrderCreationOutboxPublisher publisher) { - this.publisher = publisher; - } - - @Scheduled(fixedDelayString = "${book.creation.outbox-delay-ms:3000}") - public void publish() { publisher.publishPending(100); } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java deleted file mode 100644 index c0c01ef..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/job/OrderRecoveryScheduler.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.arch.policy.book.infrastructure.job; - -import com.arch.policy.book.application.RecoveryTaskProcessor; -import org.springframework.scheduling.annotation.Scheduled; - -public final class OrderRecoveryScheduler { - private final RecoveryTaskProcessor processor; - - public OrderRecoveryScheduler(RecoveryTaskProcessor processor) { this.processor = processor; } - - @Scheduled(fixedDelayString = "${book.recovery.scan-delay-ms:5000}") - public void scan() { processor.processBatch(System.currentTimeMillis(), 100); } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java deleted file mode 100644 index 5895766..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryBookOrderStore.java +++ /dev/null @@ -1,111 +0,0 @@ -package com.arch.policy.book.infrastructure.repository; - -import com.arch.policy.book.application.BookOrderStore; -import com.arch.policy.book.application.LocalCreateResult; -import com.arch.policy.book.application.OrderOutboxMessage; -import com.arch.policy.book.application.OrderCreationOutboxMessage; -import com.arch.policy.book.application.OrderStateHistory; -import com.arch.policy.book.application.TransitionCommitResult; -import com.arch.policy.book.domain.BookOrder; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public final class InMemoryBookOrderStore implements BookOrderStore { - private final Map orders = new HashMap(); - private final Map requestIndexes = new HashMap(); - private final Map eventResults = new HashMap(); - private final Map creationOutbox = - new HashMap(); - private final List histories = new ArrayList(); - private final List outboxMessages = new ArrayList(); - private int findByRequestIdCalls; - - @Override public synchronized BookOrder findByOrderNo(String orderNo) { - return copy(orders.get(orderNo)); - } - - @Override public synchronized BookOrder findByRequestId(String requestId) { - findByRequestIdCalls++; - String orderNo = requestIndexes.get(requestId); - return orderNo == null ? null : copy(orders.get(orderNo)); - } - - @Override public synchronized BookOrder findByEventId(String eventId) { - return copy(eventResults.get(eventId)); - } - - @Override public synchronized LocalCreateResult createOrder( - BookOrder order, OrderCreationOutboxMessage outboxMessage) { - String existingOrderNo = requestIndexes.get(order.getRequestId()); - if (existingOrderNo != null) { - return LocalCreateResult.duplicate(copy(orders.get(existingOrderNo))); - } - orders.put(order.getOrderNo(), order.copy()); - requestIndexes.put(order.getRequestId(), order.getOrderNo()); - creationOutbox.put(outboxMessage.getMessageId(), outboxMessage); - return LocalCreateResult.created(order.copy()); - } - - @Override public synchronized List findUnpublishedCreationOutbox( - int limit) { - List result = new ArrayList(); - for (OrderCreationOutboxMessage message : creationOutbox.values()) { - if (result.size() >= limit) break; - if (!message.isPublished()) result.add(message); - } - return result; - } - - @Override public synchronized void markCreationOutboxPublished(String messageId) { - OrderCreationOutboxMessage message = creationOutbox.get(messageId); - if (message != null) message.markPublished(); - } - - @Override public synchronized TransitionCommitResult transit( - BookOrder order, long expectedVersion, OrderStateHistory history, - OrderOutboxMessage outboxMessage) { - BookOrder processed = eventResults.get(history.getEventId()); - if (processed != null) return TransitionCommitResult.duplicate(processed.copy()); - - BookOrder current = orders.get(order.getOrderNo()); - if (current == null || current.getVersion() != expectedVersion - || order.getVersion() != expectedVersion + 1) { - throw new IllegalStateException("concurrent order update: " + order.getOrderNo()); - } - persist(order, history, outboxMessage); - return TransitionCommitResult.committed(order.copy()); - } - - public synchronized List historiesOf(String orderNo) { - List result = new ArrayList(); - for (OrderStateHistory history : histories) { - if (orderNo.equals(history.getOrderNo())) result.add(history); - } - return Collections.unmodifiableList(result); - } - - public synchronized int getFindByRequestIdCalls() { return findByRequestIdCalls; } - - public synchronized List unpublishedOutboxMessages() { - List result = new ArrayList(); - for (OrderOutboxMessage message : outboxMessages) { - if (!message.isPublished()) result.add(message); - } - return Collections.unmodifiableList(result); - } - - private void persist(BookOrder order, OrderStateHistory history, OrderOutboxMessage outboxMessage) { - BookOrder snapshot = order.copy(); - orders.put(order.getOrderNo(), snapshot); - eventResults.put(history.getEventId(), snapshot.copy()); - histories.add(history); - outboxMessages.add(outboxMessage); - } - - private static BookOrder copy(BookOrder order) { return order == null ? null : order.copy(); } - -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java deleted file mode 100644 index 00c7db5..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryFailedPostActionStore.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.arch.policy.book.infrastructure.repository; - -import com.arch.policy.book.application.RetryablePostTransitionExecutor.FailedPostActionStore; -import com.arch.policy.book.domain.TransitionExecution; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public final class InMemoryFailedPostActionStore implements FailedPostActionStore { - private final List failures = new ArrayList(); - - @Override public synchronized void record(TransitionExecution execution, RuntimeException failure) { - failures.add(new FailedPostAction(execution.getContext().getEventId(), - execution.getContext().getOrder().getOrderNo(), failure.getMessage())); - } - - public synchronized List all() { - return Collections.unmodifiableList(new ArrayList(failures)); - } - - public static final class FailedPostAction { - private final String eventId; - private final String orderNo; - private final String reason; - - public FailedPostAction(String eventId, String orderNo, String reason) { - this.eventId = eventId; - this.orderNo = orderNo; - this.reason = reason; - } - - public String getEventId() { return eventId; } - public String getOrderNo() { return orderNo; } - public String getReason() { return reason; } - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java deleted file mode 100644 index 25124df..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryOrderWorkflowTaskStore.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.arch.policy.book.infrastructure.repository; - -import com.arch.policy.book.application.OrderWorkflowTask; -import com.arch.policy.book.application.OrderWorkflowTaskStore; - -import java.util.LinkedHashMap; -import java.util.Map; - -public final class InMemoryOrderWorkflowTaskStore implements OrderWorkflowTaskStore { - private final Map tasks = new LinkedHashMap(); - - @Override public synchronized void saveIfAbsent(OrderWorkflowTask task) { - if (!tasks.containsKey(task.getUniqueKey())) tasks.put(task.getUniqueKey(), task); - } - - public synchronized int size() { return tasks.size(); } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java deleted file mode 100644 index c50d22b..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/repository/InMemoryRecoveryTaskStore.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.arch.policy.book.infrastructure.repository; - -import com.arch.policy.book.application.RecoveryTask; -import com.arch.policy.book.application.RecoveryTaskStore; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -public final class InMemoryRecoveryTaskStore implements RecoveryTaskStore { - private final Map tasks = new LinkedHashMap(); - - @Override public synchronized RecoveryTask saveIfAbsent(RecoveryTask task) { - RecoveryTask existing = tasks.get(task.getUniqueKey()); - if (existing != null) return existing.copy(); - tasks.put(task.getUniqueKey(), task.copy()); - return task.copy(); - } - - @Override public synchronized List findExecutable(long nowMillis, int limit) { - List result = new ArrayList(); - for (RecoveryTask task : tasks.values()) { - if (result.size() >= limit) break; - if ((task.getStatus() == RecoveryTask.RecoveryStatus.PENDING - || task.getStatus() == RecoveryTask.RecoveryStatus.RETRYING) - && task.getNextAttemptAtMillis() <= nowMillis) result.add(task.copy()); - } - return result; - } - - @Override public synchronized void save(RecoveryTask task) { - if (!tasks.containsKey(task.getUniqueKey())) { - throw new IllegalArgumentException("recovery task not found: " + task.getUniqueKey()); - } - tasks.put(task.getUniqueKey(), task.copy()); - } - - public synchronized int size() { return tasks.size(); } - - public synchronized RecoveryTask find(String uniqueKey) { - RecoveryTask task = tasks.get(uniqueKey); - return task == null ? null : task.copy(); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java deleted file mode 100644 index 869f952..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/rpc/DubboBookOrderService.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.arch.policy.book.infrastructure.rpc; - -import com.arch.policy.api.book.BookOrderRpcService; -import com.arch.policy.book.application.BookOrderApplicationService; -import com.arch.policy.book.application.OrderEventApplicationService; -import com.arch.policy.common.book.BookOrderRequest; -import com.arch.policy.common.book.BookOrderResponse; -import com.arch.policy.common.book.OrderEventRequest; -import org.apache.dubbo.config.annotation.DubboService; - -@DubboService(version = "1.0.0", timeout = 3000) -public final class DubboBookOrderService implements BookOrderRpcService { - private final BookOrderApplicationService applicationService; - private final OrderEventApplicationService eventService; - - public DubboBookOrderService(BookOrderApplicationService applicationService, - OrderEventApplicationService eventService) { - this.applicationService = applicationService; - this.eventService = eventService; - } - - @Override public BookOrderResponse createOrder(BookOrderRequest request) { - return applicationService.createOrder(request); - } - - @Override public BookOrderResponse fireEvent(OrderEventRequest request) { - return eventService.fireEvent(request); - } -} diff --git a/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java deleted file mode 100644 index 1c22b60..0000000 --- a/architecture/src/main/java/com/arch/policy/book/infrastructure/seata/SeataOrderCreationSaga.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.arch.policy.book.infrastructure.seata; - -import com.arch.policy.book.application.OrderCreationSaga; -import io.seata.saga.engine.StateMachineEngine; -import io.seata.saga.statelang.domain.StateMachineInstance; - -import java.util.HashMap; -import java.util.Map; - -public final class SeataOrderCreationSaga implements OrderCreationSaga { - public static final String STATE_MACHINE_NAME = "BookOrderCreationSaga"; - private final StateMachineEngine stateMachineEngine; - private final String tenantId; - - public SeataOrderCreationSaga(StateMachineEngine stateMachineEngine, String tenantId) { - this.stateMachineEngine = stateMachineEngine; - this.tenantId = tenantId; - } - - /** - * 启动订单创建 Saga:调用 GDS 占编,并根据结果推进订单状态或登记补偿任务。 - */ - @Override public SagaStartResult start(String orderNo, String promotionId) { - // Saga 状态节点通过上下文读取订单号和促销活动 ID,作为服务方法的入参。 - Map context = new HashMap(); - context.put("orderNo", orderNo); - // Seata 表达式和服务参数统一使用空字符串,避免 null 在状态机流转中产生歧义。 - context.put("promotionId", promotionId == null ? "" : promotionId); - - // orderNo 作为业务键标识该订单的 Saga 实例,便于恢复任务关联同一笔订单流程。 - StateMachineInstance instance = stateMachineEngine.startWithBusinessKey( - STATE_MACHINE_NAME, tenantId, orderNo, context); - - // 向上层返回 Saga 实例 ID,以及状态机是否仍处于运行中。 - return new SagaStartResult(instance.getId(), instance.isRunning()); - } -} diff --git a/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java b/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java deleted file mode 100644 index 08b6962..0000000 --- a/architecture/src/main/java/com/arch/policy/common/book/BookOrderRequest.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.arch.policy.common.book; - -import java.io.Serializable; -import java.math.BigDecimal; - -public final class BookOrderRequest implements Serializable { - private static final long serialVersionUID = 1L; - private String requestId; - private String customerId; - private String productId; - private int quantity; - private BigDecimal amount; - private String promotionId; - - public String getRequestId() { return requestId; } - public void setRequestId(String requestId) { this.requestId = requestId; } - public String getCustomerId() { return customerId; } - public void setCustomerId(String customerId) { this.customerId = customerId; } - public String getProductId() { return productId; } - public void setProductId(String productId) { this.productId = productId; } - public int getQuantity() { return quantity; } - public void setQuantity(int quantity) { this.quantity = quantity; } - public BigDecimal getAmount() { return amount; } - public void setAmount(BigDecimal amount) { this.amount = amount; } - public String getPromotionId() { return promotionId; } - public void setPromotionId(String promotionId) { this.promotionId = promotionId; } -} diff --git a/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java b/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java deleted file mode 100644 index 4845559..0000000 --- a/architecture/src/main/java/com/arch/policy/common/book/BookOrderResponse.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.arch.policy.common.book; - -import java.io.Serializable; - -public final class BookOrderResponse implements Serializable { - private static final long serialVersionUID = 1L; - private final String orderNo; - private final String state; - private final long version; - - public BookOrderResponse(String orderNo, String state, long version) { - this.orderNo = orderNo; - this.state = state; - this.version = version; - } - - public String getOrderNo() { return orderNo; } - public String getState() { return state; } - public long getVersion() { return version; } -} diff --git a/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java b/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java deleted file mode 100644 index 0e39d3a..0000000 --- a/architecture/src/main/java/com/arch/policy/common/book/OrderEventRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.arch.policy.common.book; - -import java.io.Serializable; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public final class OrderEventRequest implements Serializable { - private static final long serialVersionUID = 1L; - private String eventId; - private String orderNo; - private String event; - private String operator; - private long expectedVersion; - private Map attributes = new HashMap(); - - public String getEventId() { return eventId; } - public void setEventId(String eventId) { this.eventId = eventId; } - public String getOrderNo() { return orderNo; } - public void setOrderNo(String orderNo) { this.orderNo = orderNo; } - public String getEvent() { return event; } - public void setEvent(String event) { this.event = event; } - public String getOperator() { return operator; } - public void setOperator(String operator) { this.operator = operator; } - public long getExpectedVersion() { return expectedVersion; } - public void setExpectedVersion(long expectedVersion) { this.expectedVersion = expectedVersion; } - public Map getAttributes() { return Collections.unmodifiableMap(attributes); } - public void setAttributes(Map attributes) { - this.attributes = attributes == null - ? new HashMap() : new HashMap(attributes); - } -} diff --git a/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java deleted file mode 100644 index 3dfe543..0000000 --- a/architecture/src/main/java/com/arch/policy/common/config/BookOrderConfiguration.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.arch.policy.common.config; - -import com.arch.policy.book.application.BookOrderApplicationService; -import com.arch.policy.book.application.BookOrderStore; -import com.arch.policy.book.application.CompensationTaskService; -import com.arch.policy.book.application.CreateOrderResultCache; -import com.arch.policy.book.application.DirectOrderCreationSaga; -import com.arch.policy.book.application.GdsBookingGateway; -import com.arch.policy.book.application.GdsReconciliationService; -import com.arch.policy.book.application.OrderCreationSaga; -import com.arch.policy.book.application.OrderCreationOutboxPublisher; -import com.arch.policy.book.application.OrderCreationSagaStateServices; -import com.arch.policy.book.application.OrderEventApplicationService; -import com.arch.policy.book.application.OrderWorkflowTaskStore; -import com.arch.policy.book.application.PaymentGateway; -import com.arch.policy.book.application.PostTransitionExecutor; -import com.arch.policy.book.application.PromotionStockGateway; -import com.arch.policy.book.application.RecoveryTaskProcessor; -import com.arch.policy.book.application.RecoveryTaskStore; -import com.arch.policy.book.application.RetryablePostTransitionExecutor; -import com.arch.policy.book.domain.DefaultOrderTransitions; -import com.arch.policy.book.domain.OrderStateMachine; -import com.arch.policy.book.infrastructure.demo.DemoGdsBookingGateway; -import com.arch.policy.book.infrastructure.job.OrderRecoveryScheduler; -import com.arch.policy.book.infrastructure.job.OrderCreationOutboxScheduler; -import com.arch.policy.book.infrastructure.repository.InMemoryBookOrderStore; -import com.arch.policy.book.infrastructure.repository.InMemoryFailedPostActionStore; -import com.arch.policy.book.infrastructure.repository.InMemoryOrderWorkflowTaskStore; -import com.arch.policy.book.infrastructure.repository.InMemoryRecoveryTaskStore; -import com.arch.policy.book.infrastructure.cache.DefaultCreateOrderResultCache; -import com.arch.policy.common.infrastructure.redis.RedissonDistributedLock; -import com.arch.policy.common.infrastructure.redis.RedissonRedisClient; -import com.arch.policy.common.lock.DistributedLock; -import com.arch.policy.common.redis.RedisClient; -import com.arch.policy.book.infrastructure.seata.SeataOrderCreationSaga; -import io.seata.saga.engine.StateMachineEngine; -import org.redisson.api.RedissonClient; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; - -@Configuration -@EnableScheduling -public class BookOrderConfiguration { - @Bean public OrderStateMachine orderStateMachine() { - return new OrderStateMachine(DefaultOrderTransitions.definitions()); - } - - @Bean public BookOrderStore bookOrderStore() { return new InMemoryBookOrderStore(); } - @Bean public DistributedLock distributedLock(RedissonClient redisson) { - return new RedissonDistributedLock(redisson); - } - @Bean public RedisClient redisClient(RedissonClient redisson) { - return new RedissonRedisClient(redisson); - } - @Bean public CreateOrderResultCache createOrderResultCache(RedisClient redisClient) { - return new DefaultCreateOrderResultCache(redisClient); - } - @Bean public RecoveryTaskStore recoveryTaskStore() { return new InMemoryRecoveryTaskStore(); } - @Bean public OrderWorkflowTaskStore orderWorkflowTaskStore() { - return new InMemoryOrderWorkflowTaskStore(); - } - @Bean public InMemoryFailedPostActionStore failedPostActionStore() { - return new InMemoryFailedPostActionStore(); - } - @Bean public PostTransitionExecutor postTransitionExecutor( - InMemoryFailedPostActionStore failedPostActionStore) { - return new RetryablePostTransitionExecutor(failedPostActionStore); - } - @Bean public OrderEventApplicationService orderEventApplicationService( - BookOrderStore store, OrderStateMachine stateMachine, - PostTransitionExecutor postTransitionExecutor, - CompensationTaskService compensationTaskService, - CreateOrderResultCache resultCache) { - return new OrderEventApplicationService(store, stateMachine, postTransitionExecutor, - compensationTaskService, resultCache); - } - @Bean public GdsBookingGateway gdsBookingGateway() { return new DemoGdsBookingGateway(); } - @Bean public PromotionStockGateway promotionStockGateway() { - return (orderNo, promotionId) -> { }; - } - @Bean public PaymentGateway paymentGateway() { return (orderNo, paymentNo) -> { }; } - @Bean public CompensationTaskService compensationTaskService(RecoveryTaskStore store) { - return new CompensationTaskService(store); - } - @Bean(name = "orderCreationSagaStateServices") - public OrderCreationSagaStateServices orderCreationSagaStateServices( - BookOrderStore store, GdsBookingGateway gdsGateway, - OrderEventApplicationService eventService, CompensationTaskService compensationService, - OrderWorkflowTaskStore workflowTaskStore) { - return new OrderCreationSagaStateServices(store, gdsGateway, eventService, - compensationService, workflowTaskStore); - } - @Bean public OrderCreationSaga orderCreationSaga( - ObjectProvider engineProvider, - OrderCreationSagaStateServices services, - @Value("${book.seata.tenant-id:book}") String tenantId) { - StateMachineEngine engine = engineProvider.getIfAvailable(); - return engine == null ? new DirectOrderCreationSaga(services) - : new SeataOrderCreationSaga(engine, tenantId); - } - @Bean public BookOrderApplicationService bookOrderApplicationService( - BookOrderStore store, OrderCreationOutboxPublisher outboxPublisher, - DistributedLock distributedLock, - CreateOrderResultCache resultCache) { - return new BookOrderApplicationService(store, outboxPublisher, distributedLock, - resultCache); - } - @Bean public OrderCreationOutboxPublisher orderCreationOutboxPublisher( - BookOrderStore store, OrderCreationSaga saga) { - return new OrderCreationOutboxPublisher(store, saga); - } - @Bean public OrderCreationOutboxScheduler orderCreationOutboxScheduler( - OrderCreationOutboxPublisher publisher) { - return new OrderCreationOutboxScheduler(publisher); - } - @Bean public GdsReconciliationService gdsReconciliationService( - GdsBookingGateway gdsGateway, OrderCreationSagaStateServices sagaStates) { - return new GdsReconciliationService(gdsGateway, sagaStates); - } - @Bean public RecoveryTaskProcessor recoveryTaskProcessor( - RecoveryTaskStore taskStore, PromotionStockGateway promotionGateway, - GdsBookingGateway gdsGateway, PaymentGateway paymentGateway, - GdsReconciliationService reconciliationService) { - return new RecoveryTaskProcessor(taskStore, promotionGateway, gdsGateway, - paymentGateway, reconciliationService); - } - @Bean public OrderRecoveryScheduler orderRecoveryScheduler(RecoveryTaskProcessor processor) { - return new OrderRecoveryScheduler(processor); - } -} diff --git a/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java b/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java index 2aa68e8..94cd985 100644 --- a/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java +++ b/architecture/src/main/java/com/arch/policy/common/config/SeataSagaConfiguration.java @@ -28,8 +28,8 @@ public ThreadPoolExecutor seataSagaExecutor() { public DbStateMachineConfig dbStateMachineConfig( DataSource dataSource, ThreadPoolExecutor seataSagaExecutor, @Value("${spring.application.name:architecture}") String applicationId, - @Value("${book.seata.tx-service-group:book-saga-group}") String txServiceGroup, - @Value("${book.seata.tenant-id:book}") String tenantId) { + @Value("${order.seata.tx-service-group:order-saga-group}") String txServiceGroup, + @Value("${order.seata.tenant-id:order}") String tenantId) { DbStateMachineConfig config = new DbStateMachineConfig(); config.setDataSource(dataSource); config.setApplicationId(applicationId); @@ -37,7 +37,7 @@ public DbStateMachineConfig dbStateMachineConfig( config.setDefaultTenantId(tenantId); config.setThreadPoolExecutor(seataSagaExecutor); config.setAutoRegisterResources(true); - config.setResources(new String[] { "classpath*:statelang/book_order_creation_saga.json" }); + config.setResources(new String[] { "classpath*:statelang/order_creation_saga.json" }); config.setSagaJsonParser("jackson"); config.setSagaBranchRegisterEnable(true); return config; diff --git a/architecture/src/main/java/com/arch/policy/order/cache/OrderCacheUtil.java b/architecture/src/main/java/com/arch/policy/order/cache/OrderCacheUtil.java new file mode 100644 index 0000000..6a2e363 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/cache/OrderCacheUtil.java @@ -0,0 +1,47 @@ +package com.arch.policy.order.cache; + +import com.arch.policy.order.model.OrderResponse; +import org.redisson.api.RedissonClient; +import org.redisson.client.RedisException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * @Author : haiyang.luo + * @Date : 2026/7/22 15:58 + * @Description : + */ +@Component +public class OrderCacheUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(OrderCacheUtil.class); + + @Autowired + private RedissonClient redissonClient; + + public OrderResponse getCache(String orderSerialNo) { + try { + return redissonClient.getBucket(cacheKey(orderSerialNo)).get(); + } catch (RedisException redisFailure) { + LOGGER.warn("Create-order result cache unavailable, orderSerialNo={}", orderSerialNo, + redisFailure); + return null; + } + } + + public void setCache(String orderSerialNo, OrderResponse response) { + try { + redissonClient.getBucket(cacheKey(orderSerialNo)).set(response, 600_000L, + java.util.concurrent.TimeUnit.MILLISECONDS); + } catch (RedisException redisFailure) { + LOGGER.warn("Create-order result cache unavailable, orderSerialNo={}", orderSerialNo, + redisFailure); + } + } + + private String cacheKey(String orderSerialNo) { + return "order:create:result:" + orderSerialNo; + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/cache/OrderLockUtil.java b/architecture/src/main/java/com/arch/policy/order/cache/OrderLockUtil.java new file mode 100644 index 0000000..2f1bd36 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/cache/OrderLockUtil.java @@ -0,0 +1,45 @@ +package com.arch.policy.order.cache; + +import org.redisson.api.RLock; +import org.redisson.api.RedissonClient; +import org.redisson.client.RedisException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.concurrent.TimeUnit; + +@Component +public class OrderLockUtil { + private static final Logger LOGGER = LoggerFactory.getLogger(OrderLockUtil.class); + + @Resource + private RedissonClient redissonClient; + + public boolean tryLock(String orderSerialNo) { + RLock lock = redissonClient.getLock("order:create:lock:" + orderSerialNo); + try { + // 不设置固定过期时间,使用Redisson watchdog自动续期;只锁本地建单,不锁三方调用。 + return lock.tryLock(300L, TimeUnit.MILLISECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return false; + } catch (RedisException redisFailure) { + LOGGER.warn("Create-order lock unavailable, orderSerialNo={}", orderSerialNo, + redisFailure); + // Redis故障时继续执行,由订单库唯一索引保证最终幂等。 + return true; + } + } + + public void unlock(String orderSerialNo) { + RLock lock = redissonClient.getLock("order:create:lock:" + orderSerialNo); + try { + if (lock.isHeldByCurrentThread()) lock.unlock(); + } catch (RedisException redisFailure) { + LOGGER.warn("Release create-order lock failed, orderSerialNo={}", orderSerialNo, + redisFailure); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/InventoryTransaction.java b/architecture/src/main/java/com/arch/policy/order/model/InventoryTransaction.java new file mode 100644 index 0000000..40231bc --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/InventoryTransaction.java @@ -0,0 +1,23 @@ +package com.arch.policy.order.model; + +/** 库存库中的幂等扣减流水。 */ +public class InventoryTransaction { + private final String tradeOrderSerialNo; + private final String skuId; + private final int quantity; + private InventoryTransactionStatus status; + + public InventoryTransaction(String tradeOrderSerialNo, String skuId, int quantity, + InventoryTransactionStatus status) { + this.tradeOrderSerialNo = tradeOrderSerialNo; + this.skuId = skuId; + this.quantity = quantity; + this.status = status; + } + + public String getTradeOrderSerialNo() { return tradeOrderSerialNo; } + public String getSkuId() { return skuId; } + public int getQuantity() { return quantity; } + public InventoryTransactionStatus getStatus() { return status; } + public void setStatus(InventoryTransactionStatus status) { this.status = status; } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/InventoryTransactionStatus.java b/architecture/src/main/java/com/arch/policy/order/model/InventoryTransactionStatus.java new file mode 100644 index 0000000..e55c393 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/InventoryTransactionStatus.java @@ -0,0 +1,25 @@ +package com.arch.policy.order.model; + +/** 库存库流水状态;库存库是该状态的权威来源。 */ +public enum InventoryTransactionStatus { + /** 正在执行库存扣减,最终结果尚未确定。 */ + DEDUCTING, + + /** 库存已经扣减,等待订单创建结果决定确认或返还。 */ + DEDUCTED, + + /** 三方订单创建成功,库存扣减已经最终确认。 */ + CONFIRMED, + + /** 正在执行库存返还,最终结果尚未确定。 */ + RETURNING, + + /** 库存已经幂等返还。 */ + RETURNED, + + /** 可用库存不足,未执行库存扣减。 */ + INSUFFICIENT, + + /** 库存操作超时或异常,需要查询库存流水确认结果。 */ + UNKNOWN +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/OrderRequest.java b/architecture/src/main/java/com/arch/policy/order/model/OrderRequest.java new file mode 100644 index 0000000..5107143 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/OrderRequest.java @@ -0,0 +1,65 @@ +package com.arch.policy.order.model; + +import java.io.Serializable; + +/** + * @Author : haiyang.luo + * @Date : 2026/7/22 15:46 + * @Description : + */ +public class OrderRequest implements Serializable { + + private static final long serialVersionUID = 6809741055680415850L; + + /** + * 主订单 + */ + private String orderSerialNo; + + /** + * 供应商ID + */ + private String supplierId; + + /** + * 商品ID + */ + private String skuId; + + /** + * 下单数量 + */ + private int quantity; + + public String getOrderSerialNo() { + return orderSerialNo; + } + + public void setOrderSerialNo(String orderSerialNo) { + this.orderSerialNo = orderSerialNo; + } + + public String getSupplierId() { + return supplierId; + } + + public void setSupplierId(String supplierId) { + this.supplierId = supplierId; + } + + public String getSkuId() { + return skuId; + } + + public void setSkuId(String skuId) { + this.skuId = skuId; + } + + public int getQuantity() { + return quantity; + } + + public void setQuantity(int quantity) { + this.quantity = quantity; + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/OrderResponse.java b/architecture/src/main/java/com/arch/policy/order/model/OrderResponse.java new file mode 100644 index 0000000..d4972da --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/OrderResponse.java @@ -0,0 +1,30 @@ +package com.arch.policy.order.model; + +import java.io.Serializable; + +/** + * @Author : haiyang.luo + * @Date : 2026/7/22 15:45 + * @Description : + */ +public class OrderResponse implements Serializable { + + private static final long serialVersionUID = -655337401948649968L; + + private String orderSerialNo; + private String tradeOrderSerialNo; + private String thirdPartyOrderNo; + private String status; + private String message; + + public String getOrderSerialNo() { return orderSerialNo; } + public void setOrderSerialNo(String orderSerialNo) { this.orderSerialNo = orderSerialNo; } + public String getTradeOrderSerialNo() { return tradeOrderSerialNo; } + public void setTradeOrderSerialNo(String tradeOrderSerialNo) { this.tradeOrderSerialNo = tradeOrderSerialNo; } + public String getThirdPartyOrderNo() { return thirdPartyOrderNo; } + public void setThirdPartyOrderNo(String thirdPartyOrderNo) { this.thirdPartyOrderNo = thirdPartyOrderNo; } + public String getStatus() { return status; } + public void setStatus(String status) { this.status = status; } + public String getMessage() { return message; } + public void setMessage(String message) { this.message = message; } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/OrderStatus.java b/architecture/src/main/java/com/arch/policy/order/model/OrderStatus.java new file mode 100644 index 0000000..60716ca --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/OrderStatus.java @@ -0,0 +1,25 @@ +package com.arch.policy.order.model; + +/** 只表达订单业务生命周期,不承载库存、三方或 Saga 技术状态。 */ +public enum OrderStatus { + /** 本地订单已经创建,创建流程尚未得出最终业务结果。 */ + CREATE, + + /** 订单创建成功,正在等待用户完成支付。 */ + WAIT_PAY, + + /** 支付完成,正在向供应商执行预订。 */ + BOOKING, + + /** 供应商已经明确返回预订成功。 */ + BOOKED, + + /** 供应商明确返回预订失败。 */ + BOOK_FAIL, + + /** 订单支付失败。 */ + PAY_FAIL, + + /** 订单创建失败,例如库存不足或三方创建明确失败。 */ + CREATE_FAIL +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/SupplierOrder.java b/architecture/src/main/java/com/arch/policy/order/model/SupplierOrder.java new file mode 100644 index 0000000..a282589 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/SupplierOrder.java @@ -0,0 +1,26 @@ +package com.arch.policy.order.model; + +/** 订单库中的供应商订单关联记录,与 TradeOrder 通过子订单号关联。 */ +public class SupplierOrder { + private final String tradeOrderSerialNo; + private final String supplierId; + private final String requestNo; + private String thirdPartyOrderNo; + private SupplierOrderStatus status; + + public SupplierOrder(String tradeOrderSerialNo, String supplierId, String requestNo, + SupplierOrderStatus status) { + this.tradeOrderSerialNo = tradeOrderSerialNo; + this.supplierId = supplierId; + this.requestNo = requestNo; + this.status = status; + } + + public String getTradeOrderSerialNo() { return tradeOrderSerialNo; } + public String getSupplierId() { return supplierId; } + public String getRequestNo() { return requestNo; } + public String getThirdPartyOrderNo() { return thirdPartyOrderNo; } + public void setThirdPartyOrderNo(String thirdPartyOrderNo) { this.thirdPartyOrderNo = thirdPartyOrderNo; } + public SupplierOrderStatus getStatus() { return status; } + public void setStatus(SupplierOrderStatus status) { this.status = status; } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/SupplierOrderStatus.java b/architecture/src/main/java/com/arch/policy/order/model/SupplierOrderStatus.java new file mode 100644 index 0000000..697e9ba --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/SupplierOrderStatus.java @@ -0,0 +1,28 @@ +package com.arch.policy.order.model; + +/** 本地记录的供应商订单协作状态。 */ +public enum SupplierOrderStatus { + /** 供应商订单关联记录已经创建,尚未调用三方接口。 */ + INIT, + + /** 正在使用本地子订单号作为幂等号调用三方创建接口。 */ + CREATING, + + /** 三方已经受理请求,但订单仍处于处理中。 */ + PENDING, + + /** 三方已经明确确认订单创建成功。 */ + SUCCESS, + + /** 三方已经明确确认订单创建失败。 */ + FAILED, + + /** 调用超时或响应不明确,需要主动查询三方最终状态。 */ + UNKNOWN, + + /** 正在调用三方订单取消接口。 */ + CANCELING, + + /** 三方已经明确确认订单取消成功。 */ + CANCELED +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderResult.java b/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderResult.java new file mode 100644 index 0000000..c861e4d --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderResult.java @@ -0,0 +1,14 @@ +package com.arch.policy.order.model; + +public class ThirdOrderResult { + private final String thirdPartyOrderNo; + private final ThirdOrderStatus status; + + public ThirdOrderResult(String thirdPartyOrderNo, ThirdOrderStatus status) { + this.thirdPartyOrderNo = thirdPartyOrderNo; + this.status = status; + } + + public String getThirdPartyOrderNo() { return thirdPartyOrderNo; } + public ThirdOrderStatus getStatus() { return status; } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderStatus.java b/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderStatus.java new file mode 100644 index 0000000..19e4275 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/ThirdOrderStatus.java @@ -0,0 +1,13 @@ +package com.arch.policy.order.model; + +/** 三方订单结果;超时和处理中统一按 UNKNOWN 处理,不能直接返还库存。 */ +public enum ThirdOrderStatus { + /** 三方接口明确返回业务处理成功。 */ + SUCCESS, + + /** 三方接口明确返回业务处理失败,且不会转为成功。 */ + FAILED, + + /** 三方处理中、调用超时或响应无法确认最终业务结果。 */ + UNKNOWN +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java b/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java new file mode 100644 index 0000000..65bbec8 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java @@ -0,0 +1,23 @@ +package com.arch.policy.order.model; + +/** 本地子订单的最小快照。 */ +public class TradeOrder { + private final String orderSerialNo; + private final String tradeOrderSerialNo; + private final String supplierId; + private OrderStatus status; + + public TradeOrder(String orderSerialNo, String tradeOrderSerialNo, String supplierId, + OrderStatus status) { + this.orderSerialNo = orderSerialNo; + this.tradeOrderSerialNo = tradeOrderSerialNo; + this.supplierId = supplierId; + this.status = status; + } + + public String getOrderSerialNo() { return orderSerialNo; } + public String getTradeOrderSerialNo() { return tradeOrderSerialNo; } + public String getSupplierId() { return supplierId; } + public OrderStatus getStatus() { return status; } + public void setStatus(OrderStatus status) { this.status = status; } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/InventoryTransactionService.java b/architecture/src/main/java/com/arch/policy/order/service/InventoryTransactionService.java new file mode 100644 index 0000000..7143c9b --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/InventoryTransactionService.java @@ -0,0 +1,80 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.InventoryTransaction; +import com.arch.policy.order.model.InventoryTransactionStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** 只负责库存库事务:正常扣库存并记录幂等库存流水,不使用预占库存。 */ +@Service +public class InventoryTransactionService { + private static final Logger LOGGER = LoggerFactory.getLogger(InventoryTransactionService.class); + private final ConcurrentMap transactions = + new ConcurrentHashMap(); + + public boolean deduct(String tradeOrderSerialNo, OrderRequest request) { + LOGGER.info("[库存库事务-开始] 查询库存流水, tradeOrderSerialNo={}, skuId={}", + tradeOrderSerialNo, request.getSkuId()); + LOGGER.info("[库存库] 原子扣减库存: stock = stock - {}, 条件 stock >= {}, skuId={}", + request.getQuantity(), request.getQuantity(), request.getSkuId()); + InventoryTransaction created = new InventoryTransaction(tradeOrderSerialNo, + request.getSkuId(), request.getQuantity(), InventoryTransactionStatus.DEDUCTED); + InventoryTransaction existing = transactions.putIfAbsent(tradeOrderSerialNo, created); + InventoryTransaction transaction = existing == null ? created : existing; + LOGGER.info("[库存库] 幂等写库存流水, tradeOrderSerialNo={}, status={}", + tradeOrderSerialNo, transaction.getStatus()); + LOGGER.info("[库存库事务-提交] 库存扣减和DEDUCTED流水同时提交"); + return transaction.getStatus() == InventoryTransactionStatus.DEDUCTED + || transaction.getStatus() == InventoryTransactionStatus.CONFIRMED; + } + + public void confirm(String tradeOrderSerialNo) { + InventoryTransaction transaction = required(tradeOrderSerialNo); + synchronized (transaction) { + if (transaction.getStatus() == InventoryTransactionStatus.DEDUCTED) { + transaction.setStatus(InventoryTransactionStatus.CONFIRMED); + } else if (transaction.getStatus() != InventoryTransactionStatus.CONFIRMED) { + throw new IllegalStateException("inventory cannot be confirmed from status: " + + transaction.getStatus()); + } + } + LOGGER.info("[库存库事务] 库存流水 {} -> {}, tradeOrderSerialNo={}", + InventoryTransactionStatus.DEDUCTED, InventoryTransactionStatus.CONFIRMED, + tradeOrderSerialNo); + } + + public void returnStock(String tradeOrderSerialNo) { + InventoryTransaction transaction = required(tradeOrderSerialNo); + synchronized (transaction) { + if (transaction.getStatus() == InventoryTransactionStatus.DEDUCTED) { + transaction.setStatus(InventoryTransactionStatus.RETURNED); + } else if (transaction.getStatus() != InventoryTransactionStatus.RETURNED) { + throw new IllegalStateException("inventory cannot be returned from status: " + + transaction.getStatus()); + } + } + LOGGER.info("[库存库事务-开始] 库存流水仅允许 {} -> {}, tradeOrderSerialNo={}", + InventoryTransactionStatus.DEDUCTED, InventoryTransactionStatus.RETURNED, + tradeOrderSerialNo); + LOGGER.info("[库存库] 只有流水状态更新成功才执行 stock = stock + quantity,防止重复返还"); + LOGGER.info("[库存库事务-提交] 库存返还和RETURNED流水同时提交"); + } + + public InventoryTransaction findByTradeOrderSerialNo(String tradeOrderSerialNo) { + return transactions.get(tradeOrderSerialNo); + } + + private InventoryTransaction required(String tradeOrderSerialNo) { + InventoryTransaction transaction = transactions.get(tradeOrderSerialNo); + if (transaction == null) { + throw new IllegalStateException("inventory transaction not found: " + + tradeOrderSerialNo); + } + return transaction; + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSaga.java new file mode 100644 index 0000000..8721d4d --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSaga.java @@ -0,0 +1,9 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.TradeOrder; + +/** 本地订单提交后的跨库、跨三方创建流程。 */ +public interface OrderCreationSaga { + void start(TradeOrder order, OrderRequest request); +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java new file mode 100644 index 0000000..d464d57 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java @@ -0,0 +1,74 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.SupplierOrderStatus; +import com.arch.policy.order.model.ThirdOrderResult; +import com.arch.policy.order.model.ThirdOrderStatus; +import com.arch.policy.order.model.TradeOrder; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +/** Seata 状态节点调用的短事务服务;未知三方结果不会错误返还库存。 */ +@Service("orderCreationSagaStateServices") +public final class OrderCreationSagaStateServices { + @Resource + private OrderTransactionService orderTransactionService; + @Resource + private InventoryTransactionService inventoryTransactionService; + @Resource + private SupplierOrderService supplierOrderService; + @Resource + private SupplierOrderTransactionService supplierOrderTransactionService; + @Resource + private OrderRecoveryService orderRecoveryService; + + public boolean deductStock(TradeOrder order, OrderRequest request) { + boolean deducted = inventoryTransactionService.deduct( + order.getTradeOrderSerialNo(), request); + return deducted; + } + + public ThirdOrderResult createThirdOrder(TradeOrder order, OrderRequest request) { + supplierOrderTransactionService.markCreating(order); + ThirdOrderResult result; + try { + result = supplierOrderService.createOrder(order.getTradeOrderSerialNo(), request); + } catch (RuntimeException uncertainFailure) { + // 网络异常只代表结果未知,不能当作三方明确失败并返还库存。 + result = new ThirdOrderResult(null, ThirdOrderStatus.UNKNOWN); + } + supplierOrderTransactionService.saveResult(order.getTradeOrderSerialNo(), + result.getThirdPartyOrderNo(), statusOf(result.getStatus())); + return result; + } + + public boolean completeSuccess(TradeOrder order) { + inventoryTransactionService.confirm(order.getTradeOrderSerialNo()); + orderTransactionService.updateStatus(order, OrderStatus.WAIT_PAY); + return true; + } + + public boolean completeFailure(TradeOrder order) { + inventoryTransactionService.returnStock(order.getTradeOrderSerialNo()); + orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + return true; + } + + public boolean completeStockFailure(TradeOrder order) { + orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + return true; + } + + public boolean waitForThirdResult(TradeOrder order) { + orderRecoveryService.createThirdOrderQueryTask(order); + return true; + } + + private static SupplierOrderStatus statusOf(ThirdOrderStatus status) { + if (status == ThirdOrderStatus.SUCCESS) return SupplierOrderStatus.SUCCESS; + if (status == ThirdOrderStatus.FAILED) return SupplierOrderStatus.FAILED; + return SupplierOrderStatus.UNKNOWN; + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java b/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java new file mode 100644 index 0000000..610dadd --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java @@ -0,0 +1,67 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.SupplierOrder; +import com.arch.policy.order.model.SupplierOrderStatus; +import com.arch.policy.order.model.ThirdOrderResult; +import com.arch.policy.order.model.ThirdOrderStatus; +import com.arch.policy.order.model.TradeOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +/** 对三方超时、处理中等未知结果登记查询恢复任务。 */ +@Service +public class OrderRecoveryService { + private static final Logger LOGGER = LoggerFactory.getLogger(OrderRecoveryService.class); + + @Resource + private SupplierOrderService supplierOrderService; + @Resource + private InventoryTransactionService inventoryTransactionService; + @Resource + private OrderTransactionService orderTransactionService; + @Resource + private SupplierOrderTransactionService supplierOrderTransactionService; + + public void createThirdOrderQueryTask(TradeOrder order) { + SupplierOrder supplierOrder = supplierOrderTransactionService.findByTradeOrderSerialNo( + order.getTradeOrderSerialNo()); + LOGGER.info("[订单库事务] 创建三方订单查询任务,保持库存已扣状态,不立即返还, " + + "tradeOrderSerialNo={}, thirdPartyOrderNo={}", + order.getTradeOrderSerialNo(), thirdPartyOrderNo(supplierOrder)); + } + + /** 定时任务消费查询任务时执行;这里保留成普通方法突出恢复流程。 */ + public void reconcileThirdOrder(TradeOrder order) { + SupplierOrder supplierOrder = supplierOrderTransactionService.findByTradeOrderSerialNo( + order.getTradeOrderSerialNo()); + ThirdOrderResult result = supplierOrderService.queryOrder(order.getTradeOrderSerialNo(), + thirdPartyOrderNo(supplierOrder), order.getSupplierId()); + + if (result.getStatus() == ThirdOrderStatus.SUCCESS) { + supplierOrderTransactionService.saveResult(order.getTradeOrderSerialNo(), + result.getThirdPartyOrderNo(), SupplierOrderStatus.SUCCESS); + inventoryTransactionService.confirm(order.getTradeOrderSerialNo()); + orderTransactionService.updateStatus(order, OrderStatus.WAIT_PAY); + return; + } + + if (result.getStatus() == ThirdOrderStatus.FAILED) { + supplierOrderTransactionService.saveResult(order.getTradeOrderSerialNo(), + result.getThirdPartyOrderNo(), SupplierOrderStatus.FAILED); + inventoryTransactionService.returnStock(order.getTradeOrderSerialNo()); + orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + return; + } + + LOGGER.info("[恢复任务] 三方状态仍未知,保留已扣库存并等待下次查询, tradeOrderSerialNo={}", + order.getTradeOrderSerialNo()); + } + + private static String thirdPartyOrderNo(SupplierOrder supplierOrder) { + return supplierOrder == null ? null : supplierOrder.getThirdPartyOrderNo(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderRpcServiceImpl.java b/architecture/src/main/java/com/arch/policy/order/service/OrderRpcServiceImpl.java new file mode 100644 index 0000000..47d40f0 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderRpcServiceImpl.java @@ -0,0 +1,116 @@ +package com.arch.policy.order.service; + +import com.arch.policy.api.order.OrderRpcService; +import com.arch.policy.order.cache.OrderCacheUtil; +import com.arch.policy.order.cache.OrderLockUtil; +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.OrderResponse; +import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.SupplierOrder; +import com.arch.policy.order.model.TradeOrder; +import org.apache.dubbo.config.annotation.DubboService; + +import javax.annotation.Resource; + +/** 创建本地子订单,并在本地事务提交后同步启动 Seata Saga。 */ +@DubboService(version = "1.0.0", timeout = 3000) +public class OrderRpcServiceImpl implements OrderRpcService { + @Resource + private OrderCacheUtil orderCacheUtil; + @Resource + private OrderLockUtil orderLockUtil; + @Resource + private OrderTransactionService orderTransactionService; + @Resource + private OrderCreationSaga orderCreationSaga; + @Resource + private SupplierOrderTransactionService supplierOrderTransactionService; + + @Override + public OrderResponse createOrder(OrderRequest request) { + validate(request); + String businessKey = businessKey(request); + + OrderResponse cached = orderCacheUtil.getCache(businessKey); + if (cached != null) return cached; + + if (!orderLockUtil.tryLock(businessKey)) { + return processing(request, "相同下单请求正在处理中"); + } + + try { + cached = orderCacheUtil.getCache(businessKey); + if (cached != null) return cached; + + TradeOrder order = orderTransactionService.findByRequest( + request.getOrderSerialNo(), request.getSupplierId()); + if (order != null) { + return cacheAndReturn(businessKey, order, "返回已存在的幂等订单"); + } else { + // 这里只提交订单库本地事务,不写 Outbox,也不在事务中调用库存或三方。 + order = orderTransactionService.createOrder(request); + } + + // 对同一幂等请求持锁启动,避免多个线程同时创建相同 businessKey 的 Saga。 + orderCreationSaga.start(order, request); + return cacheAndReturn(businessKey, order, message(order.getStatus())); + } finally { + orderLockUtil.unlock(businessKey); + } + } + + private OrderResponse cacheAndReturn(String businessKey, TradeOrder order, String message) { + OrderResponse result = response(order, message); + orderCacheUtil.setCache(businessKey, result); + return result; + } + + private static String message(OrderStatus status) { + if (status == OrderStatus.WAIT_PAY) return "下单成功,等待支付"; + if (status == OrderStatus.CREATE_FAIL) return "下单失败,库存已正确处理"; + return "Saga已启动,订单处理中"; + } + + private static OrderResponse processing(OrderRequest request, String message) { + OrderResponse response = new OrderResponse(); + response.setOrderSerialNo(request.getOrderSerialNo()); + response.setStatus(OrderStatus.CREATE.name()); + response.setMessage(message); + return response; + } + + private OrderResponse response(TradeOrder order, String message) { + SupplierOrder supplierOrder = supplierOrderTransactionService + .findByTradeOrderSerialNo(order.getTradeOrderSerialNo()); + OrderResponse response = new OrderResponse(); + response.setOrderSerialNo(order.getOrderSerialNo()); + response.setTradeOrderSerialNo(order.getTradeOrderSerialNo()); + response.setThirdPartyOrderNo(supplierOrder == null + ? null : supplierOrder.getThirdPartyOrderNo()); + response.setStatus(order.getStatus().name()); + response.setMessage(message); + return response; + } + + private static String businessKey(OrderRequest request) { + return request.getOrderSerialNo() + ":" + request.getSupplierId(); + } + + private static void validate(OrderRequest request) { + if (request == null) throw new IllegalArgumentException("orderRequest is required"); + if (isBlank(request.getOrderSerialNo())) { + throw new IllegalArgumentException("orderSerialNo is required"); + } + if (isBlank(request.getSupplierId())) { + throw new IllegalArgumentException("supplierId is required"); + } + if (isBlank(request.getSkuId())) throw new IllegalArgumentException("skuId is required"); + if (request.getQuantity() <= 0) { + throw new IllegalArgumentException("quantity must be positive"); + } + } + + private static boolean isBlank(String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java b/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java new file mode 100644 index 0000000..ea69784 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java @@ -0,0 +1,43 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.TradeOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.UUID; + +/** 只负责订单库的本地事务,示例使用日志代替真实DB操作。 */ +@Service +public class OrderTransactionService { + private static final Logger LOGGER = LoggerFactory.getLogger(OrderTransactionService.class); + + public TradeOrder findByRequest(String orderSerialNo, String supplierId) { + LOGGER.info("[订单库] 按 orderSerialNo + supplierId 查询幂等订单, orderSerialNo={}, supplierId={}", + orderSerialNo, supplierId); + return null; + } + + public TradeOrder createOrder(OrderRequest request) { + String tradeOrderSerialNo = newTradeOrderSerialNo(); + LOGGER.info("[订单库事务-开始] 创建子订单"); + LOGGER.info("[订单库] 插入子订单, orderSerialNo={}, tradeOrderSerialNo={}, supplierId={}, status={}", + request.getOrderSerialNo(), tradeOrderSerialNo, request.getSupplierId(), OrderStatus.CREATE); + LOGGER.info("[订单库事务-提交] 子订单提交;提交后由应用服务同步启动Seata Saga"); + return new TradeOrder(request.getOrderSerialNo(), tradeOrderSerialNo, + request.getSupplierId(), OrderStatus.CREATE); + } + + public void updateStatus(TradeOrder order, OrderStatus status) { + order.setStatus(status); + LOGGER.info("[订单库事务] CAS更新订单状态, tradeOrderSerialNo={}, status={}", + order.getTradeOrderSerialNo(), status); + } + + private static String newTradeOrderSerialNo() { + return "TO" + UUID.randomUUID().toString().replace("-", "") + .substring(0, 20).toUpperCase(); + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/SeataOrderCreationSaga.java b/architecture/src/main/java/com/arch/policy/order/service/SeataOrderCreationSaga.java new file mode 100644 index 0000000..c2f3858 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/SeataOrderCreationSaga.java @@ -0,0 +1,31 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.TradeOrder; +import io.seata.saga.engine.StateMachineEngine; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.HashMap; +import java.util.Map; + +/** 使用 tradeOrderSerialNo 作为业务幂等键启动持久化 Seata Saga。 */ +@Component +public final class SeataOrderCreationSaga implements OrderCreationSaga { + static final String STATE_MACHINE_NAME = "TradeOrderCreationSaga"; + + @Resource + private StateMachineEngine stateMachineEngine; + @Value("${order.seata.tenant-id:order}") + private String tenantId; + + @Override + public void start(TradeOrder order, OrderRequest request) { + Map context = new HashMap(); + context.put("order", order); + context.put("request", request); + stateMachineEngine.startWithBusinessKey(STATE_MACHINE_NAME, tenantId, + order.getTradeOrderSerialNo(), context); + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderService.java b/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderService.java new file mode 100644 index 0000000..81b864e --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderService.java @@ -0,0 +1,29 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.ThirdOrderResult; +import com.arch.policy.order.model.ThirdOrderStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +/** 三方订单适配器伪实现;tradeOrderSerialNo 是三方请求的稳定幂等号。 */ +@Service +public class SupplierOrderService { + private static final Logger LOGGER = LoggerFactory.getLogger(SupplierOrderService.class); + + public ThirdOrderResult createOrder(String tradeOrderSerialNo, OrderRequest request) { + LOGGER.info("[三方接口] 幂等创建订单, tradeOrderSerialNo={}, supplierId={}", + tradeOrderSerialNo, request.getSupplierId()); + // 伪代码默认模拟明确成功;真实适配器需要映射 SUCCESS、FAILED、UNKNOWN 三种结果。 + return new ThirdOrderResult("TP" + tradeOrderSerialNo, ThirdOrderStatus.SUCCESS); + } + + public ThirdOrderResult queryOrder(String tradeOrderSerialNo, String thirdPartyOrderNo, + String supplierId) { + LOGGER.info("[三方查询接口] 查询最终订单状态, tradeOrderSerialNo={}, thirdPartyOrderNo={}, supplierId={}", + tradeOrderSerialNo, thirdPartyOrderNo, supplierId); + // 伪代码默认模拟查询后成功;长期UNKNOWN需要重试,超过阈值转人工处理。 + return new ThirdOrderResult(thirdPartyOrderNo, ThirdOrderStatus.SUCCESS); + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderTransactionService.java b/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderTransactionService.java new file mode 100644 index 0000000..dd1ad7c --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/service/SupplierOrderTransactionService.java @@ -0,0 +1,52 @@ +package com.arch.policy.order.service; + +import com.arch.policy.order.model.SupplierOrder; +import com.arch.policy.order.model.SupplierOrderStatus; +import com.arch.policy.order.model.TradeOrder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** 只负责订单库中的供应商订单关联记录;生产实现应替换为数据库事务。 */ +@Service +public class SupplierOrderTransactionService { + private static final Logger LOGGER = LoggerFactory.getLogger(SupplierOrderTransactionService.class); + private final ConcurrentMap orders = + new ConcurrentHashMap(); + + public SupplierOrder markCreating(TradeOrder order) { + SupplierOrder created = new SupplierOrder(order.getTradeOrderSerialNo(), + order.getSupplierId(), order.getTradeOrderSerialNo(), SupplierOrderStatus.CREATING); + SupplierOrder existing = orders.putIfAbsent(order.getTradeOrderSerialNo(), created); + SupplierOrder supplierOrder = existing == null ? created : existing; + LOGGER.info("[订单库事务] 幂等创建供应商订单记录, tradeOrderSerialNo={}, status={}", + order.getTradeOrderSerialNo(), supplierOrder.getStatus()); + return supplierOrder; + } + + public void saveResult(String tradeOrderSerialNo, String thirdPartyOrderNo, + SupplierOrderStatus status) { + SupplierOrder supplierOrder = required(tradeOrderSerialNo); + synchronized (supplierOrder) { + supplierOrder.setThirdPartyOrderNo(thirdPartyOrderNo); + supplierOrder.setStatus(status); + } + LOGGER.info("[订单库事务] 保存供应商订单结果, tradeOrderSerialNo={}, thirdPartyOrderNo={}, status={}", + tradeOrderSerialNo, thirdPartyOrderNo, status); + } + + public SupplierOrder findByTradeOrderSerialNo(String tradeOrderSerialNo) { + return orders.get(tradeOrderSerialNo); + } + + private SupplierOrder required(String tradeOrderSerialNo) { + SupplierOrder supplierOrder = orders.get(tradeOrderSerialNo); + if (supplierOrder == null) { + throw new IllegalStateException("supplier order not found: " + tradeOrderSerialNo); + } + return supplierOrder; + } +} diff --git a/architecture/src/main/resources/statelang/book_order_creation_saga.json b/architecture/src/main/resources/statelang/book_order_creation_saga.json deleted file mode 100644 index c6284bd..0000000 --- a/architecture/src/main/resources/statelang/book_order_creation_saga.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "Name": "BookOrderCreationSaga", - "Comment": "本地订单CREATE提交后执行GDS占编,并按SUCCESS/FAIL/UNKNOWN分流", - "Version": "1.0.0", - "StartState": "ReservePnr", - "States": { - "ReservePnr": { - "Type": "ServiceTask", - "ServiceName": "orderCreationSagaStateServices", - "ServiceMethod": "reservePnr", - "Input": ["$.[orderNo]"], - "Output": {"gdsResult": "$#root"}, - "CompensateState": "CancelReservedPnr", - "Catch": [{"Exceptions": ["java.lang.Throwable"], "Next": "MarkUnknown"}], - "Next": "RouteGdsResult" - }, - "RouteGdsResult": { - "Type": "Choice", - "Choices": [ - {"Expression": "$.[gdsResult].status.name() == 'SUCCESS'", "Next": "MarkSuccess"}, - {"Expression": "$.[gdsResult].status.name() == 'FAIL'", "Next": "MarkFailure"} - ], - "Default": "MarkUnknown" - }, - "MarkSuccess": { - "Type": "ServiceTask", - "ServiceName": "orderCreationSagaStateServices", - "ServiceMethod": "markCreateSuccess", - "Input": ["$.[orderNo]", "$.[gdsResult].pnr"], - "Next": "Succeed" - }, - "MarkFailure": { - "Type": "ServiceTask", - "ServiceName": "orderCreationSagaStateServices", - "ServiceMethod": "markCreateFailure", - "Input": ["$.[orderNo]", "$.[gdsResult].errorCode", "$.[promotionId]"], - "Next": "Succeed" - }, - "MarkUnknown": { - "Type": "ServiceTask", - "ServiceName": "orderCreationSagaStateServices", - "ServiceMethod": "markCreateUnknown", - "Input": ["$.[orderNo]", "GDS_RESULT_UNKNOWN"], - "Next": "Succeed" - }, - "CancelReservedPnr": { - "Type": "ServiceTask", - "ServiceName": "orderCreationSagaStateServices", - "ServiceMethod": "enqueueCancelPnr", - "Input": ["$.[orderNo]", "", "$.[gdsResult].pnr"] - }, - "Succeed": {"Type": "Succeed"} - } -} diff --git a/architecture/src/main/resources/statelang/order_creation_saga.json b/architecture/src/main/resources/statelang/order_creation_saga.json new file mode 100644 index 0000000..4cf077f --- /dev/null +++ b/architecture/src/main/resources/statelang/order_creation_saga.json @@ -0,0 +1,68 @@ +{ + "Name": "TradeOrderCreationSaga", + "Comment": "本地子订单提交后扣库存、调用三方并按明确结果确认或返还库存", + "Version": "1.0.0", + "StartState": "DeductStock", + "States": { + "DeductStock": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "deductStock", + "Input": ["$.[order]", "$.[request]"], + "Output": {"stockDeducted": "$#root"}, + "Next": "RouteStockResult" + }, + "RouteStockResult": { + "Type": "Choice", + "Choices": [ + {"Expression": "$.[stockDeducted] == true", "Next": "CreateThirdOrder"} + ], + "Default": "CompleteStockFailure" + }, + "CreateThirdOrder": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "createThirdOrder", + "Input": ["$.[order]", "$.[request]"], + "Output": {"thirdResult": "$#root"}, + "Next": "RouteThirdResult" + }, + "RouteThirdResult": { + "Type": "Choice", + "Choices": [ + {"Expression": "$.[thirdResult].status.name() == 'SUCCESS'", "Next": "CompleteSuccess"}, + {"Expression": "$.[thirdResult].status.name() == 'FAILED'", "Next": "CompleteFailure"} + ], + "Default": "WaitForThirdResult" + }, + "CompleteSuccess": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "completeSuccess", + "Input": ["$.[order]"], + "Next": "Succeed" + }, + "CompleteFailure": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "completeFailure", + "Input": ["$.[order]"], + "Next": "Succeed" + }, + "CompleteStockFailure": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "completeStockFailure", + "Input": ["$.[order]"], + "Next": "Succeed" + }, + "WaitForThirdResult": { + "Type": "ServiceTask", + "ServiceName": "orderCreationSagaStateServices", + "ServiceMethod": "waitForThirdResult", + "Input": ["$.[order]"], + "Next": "Succeed" + }, + "Succeed": {"Type": "Succeed"} + } +} From a2982104cdf834803555825fc7cb0aadc4434afa Mon Sep 17 00:00:00 2001 From: coderbruis Date: Wed, 22 Jul 2026 23:13:19 +0800 Subject: [PATCH 37/37] =?UTF-8?q?=E4=B8=8B=E5=8D=95=E6=B5=81=E7=A8=8B?= =?UTF-8?q?=E6=8E=A5=E5=85=A5=E7=8A=B6=E6=80=81=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + .../arch/policy/order/model/OrderEvent.java | 22 ++++++++ .../policy/order/model/OrderStateMachine.java | 53 +++++++++++++++++++ .../arch/policy/order/model/TradeOrder.java | 4 +- .../OrderCreationSagaStateServices.java | 8 +-- .../order/service/OrderRecoveryService.java | 6 +-- .../service/OrderTransactionService.java | 14 +++-- 7 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 architecture/src/main/java/com/arch/policy/order/model/OrderEvent.java create mode 100644 architecture/src/main/java/com/arch/policy/order/model/OrderStateMachine.java diff --git a/.gitignore b/.gitignore index 3540bb2..0d80b28 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ Spring-Boot/target Spring-Netty/target Spring-Security/target rocketmqdemo/target +architecture/target # .DO_Store .DS_Store diff --git a/architecture/src/main/java/com/arch/policy/order/model/OrderEvent.java b/architecture/src/main/java/com/arch/policy/order/model/OrderEvent.java new file mode 100644 index 0000000..dc14615 --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/OrderEvent.java @@ -0,0 +1,22 @@ +package com.arch.policy.order.model; + +/** 驱动订单业务状态迁移的业务事件。 */ +public enum OrderEvent { + /** 库存和三方创建流程均已成功,订单进入待支付。 */ + CREATE_SUCCEEDED, + + /** 库存不足或三方创建明确失败,订单创建失败。 */ + CREATE_FAILED, + + /** 支付成功,订单开始执行供应商预订。 */ + PAY_SUCCEEDED, + + /** 支付明确失败。 */ + PAY_FAILED, + + /** 供应商明确返回预订成功。 */ + BOOK_SUCCEEDED, + + /** 供应商明确返回预订失败。 */ + BOOK_FAILED +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/OrderStateMachine.java b/architecture/src/main/java/com/arch/policy/order/model/OrderStateMachine.java new file mode 100644 index 0000000..63a51ea --- /dev/null +++ b/architecture/src/main/java/com/arch/policy/order/model/OrderStateMachine.java @@ -0,0 +1,53 @@ +package com.arch.policy.order.model; + +import org.springframework.stereotype.Component; + +/** 集中维护订单业务状态迁移规则,禁止应用服务直接指定目标状态。 */ +@Component +public final class OrderStateMachine { + + public OrderStatus fire(TradeOrder order, OrderEvent event) { + if (order == null) throw new IllegalArgumentException("order is required"); + if (event == null) throw new IllegalArgumentException("order event is required"); + + OrderStatus current = order.getStatus(); + OrderStatus target = target(current, event); + order.applyStatus(target); + return target; + } + + private static OrderStatus target(OrderStatus current, OrderEvent event) { + // Saga和消息可能至少投递一次;相同事件到达目标状态后按幂等成功处理。 + if (current == targetOf(event)) return current; + + switch (current) { + case CREATE: + if (event == OrderEvent.CREATE_SUCCEEDED) return OrderStatus.WAIT_PAY; + if (event == OrderEvent.CREATE_FAILED) return OrderStatus.CREATE_FAIL; + break; + case WAIT_PAY: + if (event == OrderEvent.PAY_SUCCEEDED) return OrderStatus.BOOKING; + if (event == OrderEvent.PAY_FAILED) return OrderStatus.PAY_FAIL; + break; + case BOOKING: + if (event == OrderEvent.BOOK_SUCCEEDED) return OrderStatus.BOOKED; + if (event == OrderEvent.BOOK_FAILED) return OrderStatus.BOOK_FAIL; + break; + default: + break; + } + throw new IllegalStateException("invalid order transition: " + current + " + " + event); + } + + private static OrderStatus targetOf(OrderEvent event) { + switch (event) { + case CREATE_SUCCEEDED: return OrderStatus.WAIT_PAY; + case CREATE_FAILED: return OrderStatus.CREATE_FAIL; + case PAY_SUCCEEDED: return OrderStatus.BOOKING; + case PAY_FAILED: return OrderStatus.PAY_FAIL; + case BOOK_SUCCEEDED: return OrderStatus.BOOKED; + case BOOK_FAILED: return OrderStatus.BOOK_FAIL; + default: throw new IllegalArgumentException("unsupported order event: " + event); + } + } +} diff --git a/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java b/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java index 65bbec8..405a5ba 100644 --- a/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java +++ b/architecture/src/main/java/com/arch/policy/order/model/TradeOrder.java @@ -19,5 +19,7 @@ public TradeOrder(String orderSerialNo, String tradeOrderSerialNo, String suppli public String getTradeOrderSerialNo() { return tradeOrderSerialNo; } public String getSupplierId() { return supplierId; } public OrderStatus getStatus() { return status; } - public void setStatus(OrderStatus status) { this.status = status; } + + /** 仅供订单状态机应用已经校验通过的目标状态。 */ + void applyStatus(OrderStatus status) { this.status = status; } } diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java index d464d57..8a41aeb 100644 --- a/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderCreationSagaStateServices.java @@ -1,7 +1,7 @@ package com.arch.policy.order.service; import com.arch.policy.order.model.OrderRequest; -import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.OrderEvent; import com.arch.policy.order.model.SupplierOrderStatus; import com.arch.policy.order.model.ThirdOrderResult; import com.arch.policy.order.model.ThirdOrderStatus; @@ -46,18 +46,18 @@ public ThirdOrderResult createThirdOrder(TradeOrder order, OrderRequest request) public boolean completeSuccess(TradeOrder order) { inventoryTransactionService.confirm(order.getTradeOrderSerialNo()); - orderTransactionService.updateStatus(order, OrderStatus.WAIT_PAY); + orderTransactionService.fireEvent(order, OrderEvent.CREATE_SUCCEEDED); return true; } public boolean completeFailure(TradeOrder order) { inventoryTransactionService.returnStock(order.getTradeOrderSerialNo()); - orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + orderTransactionService.fireEvent(order, OrderEvent.CREATE_FAILED); return true; } public boolean completeStockFailure(TradeOrder order) { - orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + orderTransactionService.fireEvent(order, OrderEvent.CREATE_FAILED); return true; } diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java b/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java index 610dadd..3dfe8eb 100644 --- a/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderRecoveryService.java @@ -1,6 +1,6 @@ package com.arch.policy.order.service; -import com.arch.policy.order.model.OrderStatus; +import com.arch.policy.order.model.OrderEvent; import com.arch.policy.order.model.SupplierOrder; import com.arch.policy.order.model.SupplierOrderStatus; import com.arch.policy.order.model.ThirdOrderResult; @@ -45,7 +45,7 @@ public void reconcileThirdOrder(TradeOrder order) { supplierOrderTransactionService.saveResult(order.getTradeOrderSerialNo(), result.getThirdPartyOrderNo(), SupplierOrderStatus.SUCCESS); inventoryTransactionService.confirm(order.getTradeOrderSerialNo()); - orderTransactionService.updateStatus(order, OrderStatus.WAIT_PAY); + orderTransactionService.fireEvent(order, OrderEvent.CREATE_SUCCEEDED); return; } @@ -53,7 +53,7 @@ public void reconcileThirdOrder(TradeOrder order) { supplierOrderTransactionService.saveResult(order.getTradeOrderSerialNo(), result.getThirdPartyOrderNo(), SupplierOrderStatus.FAILED); inventoryTransactionService.returnStock(order.getTradeOrderSerialNo()); - orderTransactionService.updateStatus(order, OrderStatus.CREATE_FAIL); + orderTransactionService.fireEvent(order, OrderEvent.CREATE_FAILED); return; } diff --git a/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java b/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java index ea69784..9948092 100644 --- a/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java +++ b/architecture/src/main/java/com/arch/policy/order/service/OrderTransactionService.java @@ -1,18 +1,23 @@ package com.arch.policy.order.service; import com.arch.policy.order.model.OrderRequest; +import com.arch.policy.order.model.OrderEvent; +import com.arch.policy.order.model.OrderStateMachine; import com.arch.policy.order.model.OrderStatus; import com.arch.policy.order.model.TradeOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import javax.annotation.Resource; import java.util.UUID; /** 只负责订单库的本地事务,示例使用日志代替真实DB操作。 */ @Service public class OrderTransactionService { private static final Logger LOGGER = LoggerFactory.getLogger(OrderTransactionService.class); + @Resource + private OrderStateMachine orderStateMachine; public TradeOrder findByRequest(String orderSerialNo, String supplierId) { LOGGER.info("[订单库] 按 orderSerialNo + supplierId 查询幂等订单, orderSerialNo={}, supplierId={}", @@ -30,10 +35,11 @@ public TradeOrder createOrder(OrderRequest request) { request.getSupplierId(), OrderStatus.CREATE); } - public void updateStatus(TradeOrder order, OrderStatus status) { - order.setStatus(status); - LOGGER.info("[订单库事务] CAS更新订单状态, tradeOrderSerialNo={}, status={}", - order.getTradeOrderSerialNo(), status); + public void fireEvent(TradeOrder order, OrderEvent event) { + OrderStatus source = order.getStatus(); + OrderStatus target = orderStateMachine.fire(order, event); + LOGGER.info("[订单库事务] 状态机迁移并CAS更新订单, tradeOrderSerialNo={}, event={}, from={}, to={}", + order.getTradeOrderSerialNo(), event, source, target); } private static String newTradeOrderSerialNo() {